Add AppVeyor & CircleCI scripts for automated Win/Linux/OSX builds

kinke · kinke · commit a4ddbea65b84 · 2017-12-10T16:09:47.000+01:00
Windows: x86 &amp; x64, Linux and OSX: x86_64 only.
2 packages per platform (enabled/disabled assertions).

Tagged builds are uploaded to the corresponding GitHub release, untagged
builds to the GitHub 'CI' release.

AppVeyor barely manages to complete in time (timeout after 1 hour),
that's why the default 7zip compression level is used and compiler-rt &amp;
libFuzzer aren't included yet.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,187 @@
+# The common steps require environment variables CI_OS and LLVM_ENABLE_ASSERTIONS.
+commonSteps: &commonSteps
+  steps:
+    # Each step starts in working dir `<root>/project` (containing the cloned LLVM repo).
+    - run:
+        name: Install dependencies
+        command: |
+          cd ..
+          if [ "$CI_OS" = "linux" ]; then
+            export DEBIAN_FRONTEND=noninteractive
+            apt-get -y update
+            apt-get -yq install software-properties-common
+            add-apt-repository -y ppa:ubuntu-toolchain-r/test
+            apt-get -y update
+            apt-get -yq install curl git-core g++-6 binutils-dev ninja-build
+            echo "export CC=gcc-6" >> $BASH_ENV
+            echo "export CXX=g++-6" >> $BASH_ENV
+            # install CMake
+            curl -L -o cmake-x64.tar.gz https://cmake.org/files/v3.10/cmake-3.10.0-Linux-x86_64.tar.gz
+            mkdir cmake-x64
+            tar -xf cmake-x64.tar.gz --strip 1 -C cmake-x64
+            echo "export PATH=$PWD/cmake-x64/bin:$PATH" >> $BASH_ENV
+          else
+            # install CMake
+            curl -L -o cmake-x64.tar.gz https://cmake.org/files/v3.10/cmake-3.10.0-Darwin-x86_64.tar.gz
+            mkdir cmake-x64
+            tar -xf cmake-x64.tar.gz --strip 3 -C cmake-x64
+            # install Ninja
+            curl -OL https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip
+            mkdir ninja
+            tar -xf ninja-mac.zip -C ninja
+            echo "export PATH=$PWD/cmake-x64/bin:$PWD/ninja:$PATH" >> $BASH_ENV
+          fi
+    - checkout
+    - run:
+        name: Checkout git submodules
+        command: git submodule update --init --recursive
+    - run:
+        name: Build LLVM
+        command: |
+          cd ..
+          echo "$PATH"
+          ninja --version
+          cmake --version
+          mkdir ninja-llvm
+          cd ninja-llvm
+          if [ "$CI_OS" = "linux" ]; then
+            extraCMakeFlags="-DLLVM_BINUTILS_INCDIR=/usr/include -DCMAKE_CXX_FLAGS=-static-libstdc++"
+          else
+            extraCMakeFlags="-DLLVM_ENABLE_LIBCXX=True -DHAVE_FUTIMENS=0"
+          fi
+          cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$(dirname $PWD)/llvm-x64 -DLLVM_TARGETS_TO_BUILD="X86;AArch64;ARM;PowerPC;NVPTX" -DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS $extraCMakeFlags $CIRCLE_WORKING_DIRECTORY
+          ninja -j3 install
+          cd ..
+    - run:
+        name: Build compiler-rt & libFuzzer
+        command: |
+          cd ..
+          LLVM_VERSION=$(llvm-x64/bin/llvm-config --version)
+          LLVM_VERSION=${LLVM_VERSION:0:5}
+          echo "$LLVM_VERSION"
+          # Download & extract compiler-rt
+          curl -OL http://releases.llvm.org/$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
+          tar -xf compiler-rt-$LLVM_VERSION.src.tar.xz --no-same-owner
+          # Build compiler-rt
+          mkdir ninja-compiler-rt
+          cd ninja-compiler-rt
+          cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$(dirname $PWD)/llvm-x64/lib/clang/$LLVM_VERSION -DLLVM_CONFIG_PATH=$(dirname $PWD)/llvm-x64/bin/llvm-config ../compiler-rt-$LLVM_VERSION.src
+          ninja -j3 install
+          cd ..
+          # Build libFuzzer
+          mkdir libFuzzer
+          cd libFuzzer
+          if [ "$CI_OS" = "linux" ]; then
+            ../project/lib/Fuzzer/build.sh
+          else
+            # OSX needs additional `-stdlib=libc++`
+            for f in ../project/lib/Fuzzer/*.cpp; do
+              ${CXX:-clang} -g -O2 -fno-omit-frame-pointer -std=c++11 $f -c -stdlib=libc++ &
+            done
+            wait
+            ar ru libFuzzer.a Fuzzer*.o
+          fi
+          cp libFuzzer.a ../llvm-x64/lib
+          cd ..
+    - run:
+        name: Pack installation dir
+        no_output_timeout: 20m
+        command: |
+          cd ..
+          mkdir artifacts
+          assertsSuffix=""
+          if [ "$LLVM_ENABLE_ASSERTIONS" = "ON" ]; then assertsSuffix="-withAsserts"; fi
+          if [ -z "$CIRCLE_TAG" ]; then
+            artifactBasename="llvm-${CIRCLE_SHA1:0:8}-$CI_OS-x86_64$assertsSuffix-$(date "+%Y%m%d")"
+          else
+            artifactBasename="llvm-${CIRCLE_TAG:5}-$CI_OS-x86_64$assertsSuffix"
+          fi
+          mv llvm-x64 $artifactBasename
+          XZ_OPT=-9 tar -cJf "artifacts/$artifactBasename.tar.xz" $artifactBasename
+    - run:
+        name: Pack source dir
+        command: |
+          cd ..
+          if [[ "$CI_OS" = "linux" && "$LLVM_ENABLE_ASSERTIONS" = "OFF" ]]; then
+            if [ -z "$CIRCLE_TAG" ]; then
+              artifactBasename="llvm-${CIRCLE_SHA1:0:8}.src"
+            else
+              artifactBasename="llvm-${CIRCLE_TAG:5}.src"
+            fi
+            XZ_OPT=-9 tar -cJf "artifacts/$artifactBasename.tar.xz" --exclude-vcs --transform=s/project/$artifactBasename/ project
+          fi
+    - store_artifacts:
+        path: ../artifacts
+    - run:
+        name: Deploy to GitHub CI release
+        command: |
+          cd ..
+          if [ "$CI_OS" = "linux" ]; then
+            curl -L -o github-release.tar.bz2 https://github.com/aktau/github-release/releases/download/v0.7.2/linux-amd64-github-release.tar.bz2
+          else
+            curl -L -o github-release.tar.bz2 https://github.com/aktau/github-release/releases/download/v0.7.2/darwin-amd64-github-release.tar.bz2
+          fi
+          tar -xf github-release.tar.bz2 --strip 3
+          if [ -z "$CIRCLE_TAG" ]; then CIRCLE_TAG="CI"; fi
+          # Note: needs GITHUB_TOKEN environment variable
+          ./github-release upload --user ldc-developers --repo llvm --tag $CIRCLE_TAG --name "$(cd artifacts && ls llvm-*x86_64*.tar.xz)" --file artifacts/llvm-*x86_64*.tar.xz
+          if [[ "$CI_OS" = "linux" && "$LLVM_ENABLE_ASSERTIONS" = "OFF" ]]; then
+            ./github-release upload --user ldc-developers --repo llvm --tag $CIRCLE_TAG --name "$(cd artifacts && ls llvm-*src.tar.xz)" --file artifacts/llvm-*src.tar.xz
+          fi
+
+version: 2
+jobs:
+  build-linux:
+    <<: *commonSteps
+    docker:
+      - image: ubuntu:14.04
+    environment:
+      - CI_OS: "linux"
+      - LLVM_ENABLE_ASSERTIONS: "OFF"
+  build-linux-withAsserts:
+    <<: *commonSteps
+    docker:
+      - image: ubuntu:14.04
+    environment:
+      - CI_OS: "linux"
+      - LLVM_ENABLE_ASSERTIONS: "ON"
+  build-osx:
+    <<: *commonSteps
+    macos:
+      xcode: "9.0"
+    environment:
+      - CI_OS: "osx"
+      - MACOSX_DEPLOYMENT_TARGET: 10.8
+      - USE_LIBCPP: "true"
+      - LLVM_ENABLE_ASSERTIONS: "OFF"
+  build-osx-withAsserts:
+    <<: *commonSteps
+    macos:
+      xcode: "9.0"
+    environment:
+      - CI_OS: "osx"
+      - MACOSX_DEPLOYMENT_TARGET: 10.8
+      - USE_LIBCPP: "true"
+      - LLVM_ENABLE_ASSERTIONS: "ON"
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build-linux:
+          # This is required to also trigger the job after pushing a tag.
+          filters:
+            tags:
+              only: /.*/
+      - build-osx:
+          filters:
+            tags:
+              only: /.*/
+      - build-linux-withAsserts:
+          filters:
+            tags:
+              only: /.*/
+      - build-osx-withAsserts:
+          filters:
+            tags:
+              only: /.*/
diff --git a/appveyor.yml b/appveyor.yml
@@ -0,0 +1,115 @@
+#---------------------------------#
+#      general configuration      #
+#---------------------------------#
+
+#version: 1.0.{build}-{branch}
+
+#---------------------------------#
+#    environment configuration    #
+#---------------------------------#
+
+environment:
+  matrix:
+    - APPVEYOR_JOB_ARCH:           x64
+      LLVM_ENABLE_ASSERTIONS:      OFF
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - APPVEYOR_JOB_ARCH:           x86
+      LLVM_ENABLE_ASSERTIONS:      OFF
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - APPVEYOR_JOB_ARCH:           x64
+      LLVM_ENABLE_ASSERTIONS:      ON
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - APPVEYOR_JOB_ARCH:           x86
+      LLVM_ENABLE_ASSERTIONS:      ON
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
+# scripts that are called at very beginning, before repo cloning
+init:
+  - git config --global core.autocrlf input
+
+clone_depth: 1
+
+# scripts that run after cloning repository
+install:
+  - cd c:\projects
+  # Fetch submodules
+  - cd llvm
+  - git submodule update --init --recursive
+  - cd ..
+  # Download & extract Ninja
+  - appveyor DownloadFile "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip" -FileName ninja.zip
+  - md ninja
+  - cd ninja
+  - 7z x ..\ninja.zip > nul
+  - cd ..
+  # Set environment variables
+  - set PATH=%CD%\ninja;%PATH%
+  - if "%APPVEYOR_BUILD_WORKER_IMAGE:~-4%" == "2017" call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%APPVEYOR_JOB_ARCH%
+  - if "%APPVEYOR_BUILD_WORKER_IMAGE:~-4%" == "2015" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %APPVEYOR_JOB_ARCH%
+  # Print environment info
+  - set
+  - msbuild /version
+  - cl
+  - cmake --version
+  - ninja --version
+
+#---------------------------------#
+#       build configuration       #
+#---------------------------------#
+
+build_script:
+  - cd c:\projects
+  - md ninja-llvm
+  - cd ninja-llvm
+  - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=c:\projects\LLVM-%APPVEYOR_JOB_ARCH% -DLLVM_USE_CRT_RELEASE=MT -DLLVM_TARGETS_TO_BUILD=X86;AArch64;ARM;PowerPC;NVPTX -DLLVM_ENABLE_ASSERTIONS=%LLVM_ENABLE_ASSERTIONS% ..\llvm
+  - ninja -j3 install
+  # TODO: Include compiler-rt and libFuzzer as soon as AppVeyor
+  # doesn't scratch the timeout limit anymore.
+
+after_build:
+  # pack installation dir & publish as artifact
+  - ps: |
+        echo 'Preparing artifact...'
+        cd "c:\projects\LLVM-$Env:APPVEYOR_JOB_ARCH"
+        del bin\ld.lld.exe
+        del bin\lld.exe
+        $assertsSuffix = ''
+        If ($Env:LLVM_ENABLE_ASSERTIONS -eq 'ON') {
+          $assertsSuffix = '-withAsserts'
+        }
+        If (Test-Path Env:APPVEYOR_REPO_TAG_NAME) {
+          $artifactFilename = "llvm-$($Env:APPVEYOR_REPO_TAG_NAME.Substring(5))-windows-$Env:APPVEYOR_JOB_ARCH$assertsSuffix.7z"
+        } Else {
+          $artifactFilename = "llvm-$($Env:APPVEYOR_REPO_COMMIT.Substring(0, 8))-windows-$Env:APPVEYOR_JOB_ARCH$assertsSuffix-$(get-date -f yyyyMMdd).7z"
+        }
+        # TODO: Use higher compression level (-mx=9) as soon as
+        # AppVeyor doesn't scratch the timeout limit anymore.
+        7z a "..\$artifactFilename" * > $null
+        cd ..
+        Push-AppveyorArtifact $artifactFilename
+
+test: off
+
+#---------------------------------#
+#     deployment configuration    #
+#---------------------------------#
+
+deploy:
+  - release: CI
+    prerelease: true
+    provider: GitHub
+    auth_token:
+      secure: qnbD8agL9mr0SFvy/sMkR2E29oQQ427T5zYwVVZkjRS3IZ361tG+9jlSiyEkyULy
+    artifact: /llvm-.*\.7z/
+    on:
+      appveyor_repo_tag: false
+  - release: $(APPVEYOR_REPO_TAG_NAME)
+    description: $(APPVEYOR_REPO_TAG_NAME)
+    prerelease: true
+    draft: true
+    provider: GitHub
+    auth_token:
+      secure: qnbD8agL9mr0SFvy/sMkR2E29oQQ427T5zYwVVZkjRS3IZ361tG+9jlSiyEkyULy
+    artifact: /llvm-.*\.7z/
+    on:
+      appveyor_repo_tag: true
