Merge pull request #189 from JuliaMath/sf/preferences

Require Julia 1.6+, use Preferences, eliminate mutable state

Author: staticfloat

.drone.jsonnet

@@ -10,7 +10,7 @@ local Pipeline(os, arch, version) = {
 	    name: "build",
 	    image: "julia:"+version,
 	    commands: [
-		"julia --project=. --check-bounds=yes --color=yes -e 'using InteractiveUtils; versioninfo(verbose=true); using Pkg; Pkg.build(); Pkg.test(coverage=true)'"
+		"julia --project=. --check-bounds=yes --color=yes -e 'using InteractiveUtils; versioninfo(verbose=true); using Pkg; Pkg.test(coverage=true)'"
 	    ]
 	}
     ],
@@ -20,7 +20,7 @@ local Pipeline(os, arch, version) = {
 };
 
 [
-    Pipeline("linux", "arm",   "1.3"),
-    Pipeline("linux", "arm64", "1.3"),
-    Pipeline("linux", "arm64", "1.5")
+    # Commenting this out because we don't have an official armv7l build yet
+    #Pipeline("linux", "arm",   "1.6"),
+    Pipeline("linux", "arm64", "1.6")
 ]

.drone.yml

@@ -1,52 +1,16 @@
 ---
 kind: pipeline
-name: linux - arm - Julia 1.3
-
-platform:
-  os: linux
-  arch: arm
-
-steps:
-- name: build
-  image: julia:1.3
-  commands:
-  - "julia --project=. --check-bounds=yes --color=yes -e 'using InteractiveUtils; versioninfo(verbose=true); using Pkg; Pkg.build(); Pkg.test(coverage=true)'"
-
-trigger:
-  branch:
-  - master
-
----
-kind: pipeline
-name: linux - arm64 - Julia 1.3
-
-platform:
-  os: linux
-  arch: arm64
-
-steps:
-- name: build
-  image: julia:1.3
-  commands:
-  - "julia --project=. --check-bounds=yes --color=yes -e 'using InteractiveUtils; versioninfo(verbose=true); using Pkg; Pkg.build(); Pkg.test(coverage=true)'"
-
-trigger:
-  branch:
-  - master
-
----
-kind: pipeline
-name: linux - arm64 - Julia 1.5
+name: linux - arm64 - Julia 1.6
 
 platform:
   os: linux
   arch: arm64
 
 steps:
 - name: build
-  image: julia:1.5
+  image: julia:1.6
   commands:
-  - "julia --project=. --check-bounds=yes --color=yes -e 'using InteractiveUtils; versioninfo(verbose=true); using Pkg; Pkg.build(); Pkg.test(coverage=true)'"
+  - "julia --project=. --check-bounds=yes --color=yes -e 'using InteractiveUtils; versioninfo(verbose=true); using Pkg; Pkg.test(coverage=true)'"
 
 trigger:
   branch:

.github/set_ci_preferences.jl

@@ -0,0 +1,8 @@
+#!/usr/bin/env julia 
+
+open(joinpath(dirname(@__DIR__), "test", "Project.toml"), "a") do io
+    println(io, """
+    [preferences.FFTW]
+    provider = "$(ARGS[1])"
+    """)
+end

.github/workflows/CI.yml

@@ -13,16 +13,15 @@ jobs:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - provider ${{ matrix.provider }} - ${{ matrix.threads }} thread(s)
     runs-on: ${{ matrix.os }}
     env:
-      JULIA_FFTW_PROVIDER: ${{ matrix.provider }}
       JULIA_NUM_THREADS: ${{ matrix.threads }}
     strategy:
       fail-fast: false
       matrix:
         provider:
-          - 'FFTW'
-          - 'MKL'
+          - 'fftw'
+          - 'mkl'
         version:
-          - '1.3'
+          - '1.6'
           - 'nightly'
         os:
           - ubuntu-latest
@@ -40,9 +39,9 @@ jobs:
             arch: x86
           # 32-bit Linux binary for MKL isn't always available, let's ignore it
           - os: ubuntu-latest
-            provider: 'MKL'
+            provider: 'mkl'
             arch: x86
-          - provider: 'MKL'
+          - provider: 'mkl'
             threads: '2'
 
     steps:
@@ -61,7 +60,8 @@ jobs:
             ${{ runner.os }}-test-${{ env.cache-name }}-
             ${{ runner.os }}-test-
             ${{ runner.os }}-
-      - uses: julia-actions/julia-buildpkg@v1
+      - name: Set Preferences
+        run: julia --project .github/set_ci_preferences.jl "${{ matrix.provider }}"
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1

Project.toml

@@ -2,22 +2,24 @@ name = "FFTW"
 uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
 version = "1.3.2"
 
+[compat]
+AbstractFFTs = "1.0"
+FFTW_jll = "3.3"
+IntelOpenMP_jll = "2018.0.3"
+MKL_jll = "2019.0.117, 2020, 2021"
+Reexport = "0.2, 1.0"
+julia = "1.6"
+
 [deps]
 AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c"
 FFTW_jll = "f5851436-0d7a-5f13-b9de-f02708fd171a"
 IntelOpenMP_jll = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 
-[compat]
-AbstractFFTs = "1.0"
-FFTW_jll = "3.3"
-IntelOpenMP_jll = "2018.0.3"
-MKL_jll = "2019.0.117, 2020, 2021"
-Reexport = "0.2, 1.0"
-julia = "1.3"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

deps/build.jl

@@ -1,6 +1,6 @@
 module FFTW
 
-using LinearAlgebra, Reexport
+using LinearAlgebra, Reexport, Preferences
 import Libdl
 @reexport using AbstractFFTs
 using Base.Threads
@@ -15,47 +15,18 @@ import AbstractFFTs: Plan, ScaledPlan,
 
 export dct, idct, dct!, idct!, plan_dct, plan_idct, plan_dct!, plan_idct!
 
-const depsfile = joinpath(dirname(@__DIR__), "deps", "deps.jl")
-if isfile(depsfile)
-    include(depsfile)
-else
-    error("FFTW is not properly installed. Please run Pkg.build(\"FFTW\") ",
-          "and restart Julia.")
-end
-
-# MKL provides its own FFTW
-const fftw_vendor = occursin("mkl_rt", libfftw3) ? :mkl : :fftw
+include("providers.jl")
 
-# Use Julia partr threading backend if present
-@static if fftw_vendor == :fftw
-    # callback function that FFTW uses to launch `num` parallel
-    # tasks (FFTW/fftw3#175):
-    function spawnloop(f::Ptr{Cvoid}, fdata::Ptr{Cvoid}, elsize::Csize_t, num::Cint, callback_data::Ptr{Cvoid})
-        @sync for i = 0:num-1
-            Threads.@spawn ccall(f, Ptr{Cvoid}, (Ptr{Cvoid},), fdata + elsize*i)
-        end
-    end
-end
-
-# If FFTW was built with threads, then they must be initialized before any FFTW planning routine.
-#   -- This initializes FFTW's threads support (defaulting to 1 thread).
-#      If this isn't called before the FFTW planner is created, then
-#      FFTW's threads algorithms won't be registered or used at all.
-#      (Previously, we called fftw_cleanup, but this invalidated existing
-#       plans, causing Base Julia issue #19892.)
 function __init__()
-    check_deps()
-    stat =  ccall((:fftw_init_threads,   libfftw3), Int32, ())
-    statf = ccall((:fftwf_init_threads, libfftw3f), Int32, ())
-    if stat == 0 || statf == 0
-        error("could not initialize FFTW threads")
+    # If someone is trying to set the provider via the old environment variable, warn them that they
+    # should instead use `set_provider!()` instead.
+    if haskey(ENV, "JULIA_FFTW_PROVIDER")
+        Base.depwarn("JULIA_FFTW_PROVIDER is deprecated; use FFTW.set_provider!() instead", :JULIA_FFTW_PROVIDER)
     end
-    @static if fftw_vendor == :fftw
-        if nthreads() > 1
-            cspawnloop = @cfunction(spawnloop, Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Cint, Ptr{Cvoid}))
-            ccall((:fftw_threads_set_callback,  libfftw3),  Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), cspawnloop, C_NULL)
-            ccall((:fftwf_threads_set_callback, libfftw3f), Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), cspawnloop, C_NULL)
-        end
+
+    # Hook FFTW threads up to our partr runtime
+    @static if fftw_provider == "fftw"
+        fftw_init_threads()
     end
 end
 
@@ -91,9 +62,7 @@ end
 include("fft.jl")
 include("dct.jl")
 
-if Base.VERSION >= v"1.4.2"     # avoid potential segfaults, see https://github.com/JuliaLang/julia/pull/35378
-    include("precompile.jl")
-    _precompile_()
-end
+include("precompile.jl")
+_precompile_()
 
 end # module

src/FFTW.jl

@@ -203,7 +203,7 @@ unsafe_set_timelimit(precision::fftwTypeSingle,seconds) =
 #   function will be documented in FFTW 3.3.4.
 
 
-@static if fftw_vendor == :mkl
+@static if fftw_provider == "mkl"
     alignment_of(A::StridedArray{<:fftwDouble}) =
         convert(Int32, convert(Int64, pointer(A)) % 16)
     alignment_of(A::StridedArray{<:fftwSingle}) =
@@ -366,7 +366,7 @@ end
 
 # The sprint_plan function was released in FFTW 3.3.4, but MKL versions
 # claiming to be FFTW 3.3.4 still don't seem to have this function.
-const has_sprint_plan = version >= v"3.3.4" && fftw_vendor == :fftw
+const has_sprint_plan = version >= v"3.3.4" && fftw_provider == "fftw"
 
 @static if has_sprint_plan
     sprint_plan_(plan::FFTWPlan{<:fftwDouble}) =

src/fft.jl

@@ -0,0 +1,80 @@
+
+function get_provider()
+    # Note: we CANNOT do something like have the `default` value be `get(ENV, "JULIA_FFTW_PROVIDER", "fftw")` here.
+    # This is because the only way the Julia knows that a default has changed is if the values on-disk change; so
+    # if your "default" value can be changed from the outside, you quickly run into cache invalidation issues.
+    # So the default here _must_ be a constant.
+    default_provider = "fftw"
+
+    # Load the preference
+    provider = @load_preference("provider", default_provider)
+
+    # Ensure the provider matches one of the ones we support
+    if provider ∉ ("fftw", "mkl")
+        @error("Invalid provider setting \"$(provider)\"; valid settings include [\"fftw\", \"mkl\"], defaulting to \"fftw\"")
+        provider = default_provider
+    end
+    return provider
+end
+
+# Read in preferences, see if any users have requested a particular backend
+const fftw_provider = get_provider()
+
+"""
+    set_provider!(provider; export_prefs::Bool = false)
+
+Convenience wrapper for setting the FFT provider.  Valid values include `"fftw"`, `"mkl"`.
+Also supports `Preferences` sentinel values `nothing` and `missing`; see the docstring for
+`Preferences.set_preferences!()` for more information on what these values mean.
+"""
+function set_provider!(provider; export_prefs::Bool = false)
+    if provider !== nothing && provider !== missing && provider ∉ ("fftw", "mkl")
+        throw(ArgumentError("Invalid provider value '$(provider)'"))
+    end
+    set_preferences!(@__MODULE__, "provider" => provider; export_prefs)
+    if provider != fftw_provider
+        # Re-fetch to get default values in the event that `nothing` or `missing` was passed in.
+        provider = get_provider()
+        @info("FFTW provider changed; restart Julia for this change to take effect", provider)
+    end
+end
+
+# If we're using fftw_jll, load it in
+@static if fftw_provider == "fftw"
+    using FFTW_jll
+
+    # callback function that FFTW uses to launch `num` parallel
+    # tasks (FFTW/fftw3#175):
+    function spawnloop(f::Ptr{Cvoid}, fdata::Ptr{Cvoid}, elsize::Csize_t, num::Cint, callback_data::Ptr{Cvoid})
+        @sync for i = 0:num-1
+            Threads.@spawn ccall(f, Ptr{Cvoid}, (Ptr{Cvoid},), fdata + elsize*i)
+        end
+    end
+
+    # If FFTW was built with threads, then they must be initialized before any FFTW planning routine.
+    #   -- This initializes FFTW's threads support (defaulting to 1 thread).
+    #      If this isn't called before the FFTW planner is created, then
+    #      FFTW's threads algorithms won't be registered or used at all.
+    #      (Previously, we called fftw_cleanup, but this invalidated existing
+    #       plans, causing Base Julia issue #19892.)
+    function fftw_init_threads()
+        stat =  ccall((:fftw_init_threads,   libfftw3), Int32, ())
+        statf = ccall((:fftwf_init_threads, libfftw3f), Int32, ())
+        if stat == 0 || statf == 0
+            error("could not initialize FFTW threads")
+        end
+
+        if nthreads() > 1
+            cspawnloop = @cfunction(spawnloop, Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Cint, Ptr{Cvoid}))
+            ccall((:fftw_threads_set_callback,  libfftw3),  Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), cspawnloop, C_NULL)
+            ccall((:fftwf_threads_set_callback, libfftw3f), Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), cspawnloop, C_NULL)
+        end
+    end
+end
+
+# If we're using MKL, load it in and set library paths appropriately.
+@static if fftw_provider == "mkl"
+    using IntelOpenMP_jll, MKL_jll
+    const libfftw3 = MKL_jll.libmkl_rt_path
+    const libfftw3f = libfftw3
+end

src/providers.jl

@@ -0,0 +1,14 @@
+# A bug in Julia 1.6.0's Pkg causes Preferences to be dropped during `Pkg.test()`, so we work around
+# it by explicitly creating a `test/Project.toml` which will correctly communicate any preferences
+# through to the child Julia process.  X-ref: https://github.com/JuliaLang/Pkg.jl/issues/2500
+
+[deps]
+AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c"
+FFTW_jll = "f5851436-0d7a-5f13-b9de-f02708fd171a"
+IntelOpenMP_jll = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"