Automatically have CI build LLVMExtra, if needed. (#318)

Author: maleadt

.github/workflows/ci.yml

@@ -6,17 +6,19 @@ on:
   pull_request:
   workflow_dispatch:
 jobs:
-  binary_test:
+  test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        version: ['1.6', '1.7', '^1.8.0-beta3', 'nightly']
+        version: ['1.6', '1.7', '1.8', 'nightly']
         os: [ubuntu-latest, macOS-latest, windows-latest]
         arch: [x64]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # build_ci.jl needs to be able to inspect the git log
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
@@ -32,55 +34,10 @@ jobs:
             ${{ runner.os }}-test-
             ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
-      - name: Run tests
-        run: |
-          julia --project -e 'using Pkg; Pkg.test(; coverage=true, julia_args=`-g2`)'
-      - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
-        with:
-          file: lcov.info
-
-  # development versions of Julia, built with assertions enabled.
-  # we also build LLVMExtra from source here.
-  source_test:
-    name: Julia ${{ matrix.branch }} - ${{ matrix.os }} - ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        branch: ['release-1.7', 'release-1.8', 'master']
-        os: [ubuntu-latest, macOS-latest]
-        arch: [x64]
-        assertions: [false]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/checkout@v2
-        with:
-          repository: 'JuliaLang/julia'
-          ref: ${{ matrix.branch }}
-          path: 'julia'
-      - name: Compile Julia
-        run: |
-          make -C julia -j $(nproc) FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 JULIA_PRECOMPILE=0
-          echo $PWD/julia/usr/bin >> $GITHUB_PATH
-      - uses: actions/cache@v1
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
-      - uses: julia-actions/julia-buildpkg@v1
       - name: Build libLLVMExtra
-        run: |
-          julia --project=deps -e 'using Pkg; Pkg.instantiate()'
-          julia --project=deps deps/build_local.jl
+        run: julia --project=deps deps/build_ci.jl
       - name: Run tests
-        run: |
-          julia --project -e 'using Pkg; Pkg.test(; coverage=true, julia_args=`-g2`)'
+        run: julia --project -e 'using Pkg; Pkg.test(; coverage=true, julia_args=`-g2`)'
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:

deps/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 CMake_jll = "3f4e10e2-61f2-5801-8945-23b9d642d0e6"
+Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 LLVMExtra_jll = "dad2f222-ce93-54a1-a47d-0025e8a3acab"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

deps/build_ci.jl

@@ -0,0 +1,54 @@
+using Pkg
+Pkg.activate(@__DIR__)
+Pkg.instantiate()
+
+using Git, Scratch, Dates
+
+LLVM = Base.UUID("929cbde3-209d-540e-8aea-75f648917ca0")
+
+# get scratch directories
+support_dir = get_scratch!(LLVM, "support")
+
+# is this a full-fledged check-out?
+if isdir(joinpath(@__DIR__), "..", ".git")
+    # determine latest change to the wrappers
+    deps_timestamp = parse(Int, read(`$(git()) -C $(@__DIR__) log -1 --format=%ct LLVMExtra`, String))
+    @info "Latest change to the wrappers: $(unix2datetime(deps_timestamp))"
+
+    # find out which version of LLVMExtra_jll we are using
+    Pkg.activate(joinpath(@__DIR__, ".."))
+    deps = collect(values(Pkg.dependencies()))
+    filter!(deps) do dep
+        dep.name == "LLVMExtra_jll"
+    end
+    library_version = only(deps).version
+    @info "LLVMExtra_jll version: $(library_version)"
+
+    # compare to the JLL's tags
+    jll_tags = mktempdir() do dir
+        if !isdir(joinpath(support_dir, ".git"))
+            run(`$(git()) clone -q https://github.com/JuliaBinaryWrappers/LLVMExtra_jll.jl $dir`)
+        else
+            run(`$(git()) -C $dir fetch -q`)
+        end
+        tags = Dict{String,Int}()
+        for line in eachline(`$(git()) -C $dir tag --format "%(refname:short) %(creatordate:unix)"`)
+            tag, timestamp = split(line)
+            tags[tag] = parse(Int, timestamp)
+        end
+        tags
+    end
+    jll_timestamp = jll_tags["LLVMExtra-v$(library_version)"]
+    @info "LLVMExtra_jll timestamp: $(unix2datetime(jll_timestamp))"
+
+    if deps_timestamp > jll_timestamp
+        @info "Wrappers have changed since the last JLL build. Building the support library locally."
+        include(joinpath(@__DIR__, "build_local.jl"))
+    else
+        @info "Wrappers have not changed since the last JLL build. Using the JLL's support library."
+    end
+else
+    @warn """LLVM.jl source code is not checked-out from Git.
+             This means we cannot check for changes, and need to unconditionally build the support library."""
+    include(joinpath(@__DIR__, "build_local.jl"))
+end

deps/build_local.jl

@@ -1,11 +1,17 @@
-# Invoke with
-# `julia --project=deps deps/build_local.jl`
+# build a local version of LLVMExtra
 
-# the pre-built LLVMExtra_jll might not be loadable on this platform
-LLVMExtra_jll = Base.UUID("dad2f222-ce93-54a1-a47d-0025e8a3acab")
+using Pkg
+Pkg.activate(@__DIR__)
+Pkg.instantiate()
+
+if haskey(ENV, "GITHUB_ACTIONS")
+    println("::warning ::Using a locally-built LLVMExtra; A bump of LLVMExtra_jll will be required before releasing LLVM.jl.")
+end
 
 using Pkg, Scratch, Preferences, Libdl, CMake_jll
 
+LLVM = Base.UUID("929cbde3-209d-540e-8aea-75f648917ca0")
+
 # 1. Ensure that an appropriate LLVM_full_jll is installed
 Pkg.activate(; temp=true)
 llvm_assertions = try
@@ -26,7 +32,7 @@ end
 LLVM_DIR = joinpath(LLVM.artifact_dir, "lib", "cmake", "llvm")
 
 # 2. Get a scratch directory
-scratch_dir = get_scratch!(LLVMExtra_jll, "build")
+scratch_dir = get_scratch!(LLVM, "build")
 isdir(scratch_dir) && rm(scratch_dir; recursive=true)
 source_dir = joinpath(@__DIR__, "LLVMExtra")