build + Kernel creation

Author: vchuravy

Project.toml

@@ -9,10 +9,12 @@ Atomix = "a9b6321e-bd34-4604-b9c9-b65b8de01458"
 EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
 GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 OpenCL_jll = "6cb37087-e8b6-5417-8430-1f242f1e46e4"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SPIRVIntrinsics = "71d1d633-e7e8-4a92-83a1-de8814b09ba8"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

src/pocl/compiler/compilation.jl

@@ -64,7 +64,7 @@ end
 
 # link into an executable kernel
 function link(@nospecialize(job::CompilerJob), compiled)
-    prog = if "cl_khr_il_program" in cl.device().extensions
+    prog = if "cl_khr_il_program" in device().extensions
         cl.Program(compiled.obj, context())
     else
         error("Your device does not support SPIR-V, which is currently required for native execution.")

src/pocl/compiler/execution.jl

@@ -184,8 +184,8 @@ end
 const clfunction_lock = ReentrantLock()
 
 function clfunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT}
-    ctx = cl.context()
-    dev = cl.device()
+    ctx = context()
+    dev = device()
 
     Base.@lock clfunction_lock begin
         # compile the function

src/pocl/nanoOpenCL.jl

@@ -3,6 +3,8 @@ module nanoOpenCL
 import OpenCL_jll
 import pocl_jll
 
+using Printf
+
 const libopencl = OpenCL_jll.libopencl # TODO directly use POCL
 
 """
@@ -302,6 +304,55 @@ const CL_DEVICE_PIPE_SUPPORT = 0x1071
 
 const CL_DEVICE_LATEST_CONFORMANCE_VERSION_PASSED = 0x1072
 
+const CL_PROGRAM_REFERENCE_COUNT = 0x1160
+
+const CL_PROGRAM_CONTEXT = 0x1161
+
+const CL_PROGRAM_NUM_DEVICES = 0x1162
+
+const CL_PROGRAM_DEVICES = 0x1163
+
+const CL_PROGRAM_SOURCE = 0x1164
+
+const CL_PROGRAM_BINARY_SIZES = 0x1165
+
+const CL_PROGRAM_BINARIES = 0x1166
+
+const CL_PROGRAM_NUM_KERNELS = 0x1167
+
+const CL_PROGRAM_KERNEL_NAMES = 0x1168
+
+const CL_PROGRAM_IL = 0x1169
+
+const CL_PROGRAM_SCOPE_GLOBAL_CTORS_PRESENT = 0x116a
+
+const CL_PROGRAM_SCOPE_GLOBAL_DTORS_PRESENT = 0x116b
+
+const CL_PROGRAM_BUILD_STATUS = 0x1181
+
+const CL_PROGRAM_BUILD_OPTIONS = 0x1182
+
+const CL_PROGRAM_BUILD_LOG = 0x1183
+
+const CL_PROGRAM_BINARY_TYPE = 0x1184
+
+const CL_PROGRAM_BUILD_GLOBAL_VARIABLE_TOTAL_SIZE = 0x1185
+
+const CL_PROGRAM_BINARY_TYPE_NONE = 0x00
+
+const CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT = 0x01
+
+const CL_PROGRAM_BINARY_TYPE_LIBRARY = 0x02
+
+const CL_PROGRAM_BINARY_TYPE_EXECUTABLE = 0x04
+
+const CL_BUILD_SUCCESS = 0
+
+const CL_BUILD_NONE = -1
+
+const CL_BUILD_ERROR = -2
+
+const CL_BUILD_IN_PROGRESS = -3
 
 function check(f)
     res = f()
@@ -365,6 +416,11 @@ const cl_context_properties = intptr_t
 
 const cl_context_info = cl_uint
 
+const cl_build_status = cl_int
+
+const cl_program_info = cl_uint
+
+const cl_program_build_info = cl_uint
 
 @checked function clGetPlatformIDs(num_entries, platforms, num_platforms)
     @ccall libopencl.clGetPlatformIDs(num_entries::cl_uint, platforms::Ptr{cl_platform_id},
@@ -414,6 +470,29 @@ end
     @ccall libopencl.clReleaseProgram(program::cl_program)::cl_int
 end
 
+@checked function clBuildProgram(program, num_devices, device_list, options, pfn_notify,
+                                 user_data)
+    @ccall libopencl.clBuildProgram(program::cl_program, num_devices::cl_uint,
+                                    device_list::Ptr{cl_device_id}, options::Ptr{Cchar},
+                                    pfn_notify::Ptr{Cvoid}, user_data::Ptr{Cvoid})::cl_int
+end
+
+@checked function clGetProgramInfo(program, param_name, param_value_size, param_value,
+                                   param_value_size_ret)
+    @ccall libopencl.clGetProgramInfo(program::cl_program, param_name::cl_program_info,
+                                      param_value_size::Csize_t, param_value::Ptr{Cvoid},
+                                      param_value_size_ret::Ptr{Csize_t})::cl_int
+end
+
+@checked function clGetProgramBuildInfo(program, device, param_name, param_value_size,
+                                        param_value, param_value_size_ret)
+    @ccall libopencl.clGetProgramBuildInfo(program::cl_program, device::cl_device_id,
+                                           param_name::cl_program_build_info,
+                                           param_value_size::Csize_t,
+                                           param_value::Ptr{Cvoid},
+                                           param_value_size_ret::Ptr{Csize_t})::cl_int
+end
+
 function clCreateKernel(program, kernel_name, errcode_ret)
     @ccall libopencl.clCreateKernel(program::cl_program, kernel_name::Ptr{Cchar},
                                     errcode_ret::Ptr{cl_int})::cl_kernel
@@ -741,6 +820,116 @@ function Program(il, ctx)
     Program(program_id)
 end
 
+#TODO: build callback...
+function build!(p::Program; options="")
+    opts = String(options)
+    ndevices = 0
+    device_ids = C_NULL
+    try
+        clBuildProgram(p, cl_uint(ndevices), device_ids, opts, C_NULL, C_NULL)
+    catch err
+        isa(err, CLError) || throw(err)
+
+        for (dev, status) in p.build_status
+            if status == CL_BUILD_ERROR
+                io = IOBuffer()
+                println(io, "Failed to compile program")
+                if p.source !== nothing
+                    println(io)
+                    println(io, "Source code:")
+                    for (i,line) in enumerate(split(p.source, "\n"))
+                        println(io, @sprintf("%s%-2d: %s", " ", i, line))
+                    end
+                end
+                println(io)
+                println(io, "Build log:")
+                println(io, strip(p.build_log[dev]))
+                error(String(take!(io)))
+            end
+        end
+    end
+    return p
+end
+
+function Base.getproperty(p::Program, s::Symbol)
+    if s == :reference_count
+        count = Ref{Cuint}()
+        clGetProgramInfo(p, CL_PROGRAM_REFERENCE_COUNT, sizeof(Cuint), count, C_NULL)
+        return Int(count[])
+    elseif s == :num_devices
+        count = Ref{Cuint}()
+        clGetProgramInfo(p, CL_PROGRAM_NUM_DEVICES, sizeof(Cuint), count, C_NULL)
+        return Int(count[])
+    elseif s == :devices
+        device_ids = Vector{cl_device_id}(undef, p.num_devices)
+        clGetProgramInfo(p, CL_PROGRAM_DEVICES, sizeof(device_ids), device_ids, C_NULL)
+        return [Device(id) for id in device_ids]
+    elseif s == :source
+        src_len = Ref{Csize_t}()
+        clGetProgramInfo(p, CL_PROGRAM_SOURCE, 0, C_NULL, src_len)
+        src_len[] <= 1 && return nothing
+        src = Vector{Cchar}(undef, src_len[])
+        clGetProgramInfo(p, CL_PROGRAM_SOURCE, src_len[], src, C_NULL)
+        return GC.@preserve src unsafe_string(pointer(src))
+    elseif s == :binary_sizes
+        sizes = Vector{Csize_t}(undef, p.num_devices)
+        clGetProgramInfo(p, CL_PROGRAM_BINARY_SIZES, sizeof(sizes), sizes, C_NULL)
+        return sizes
+    elseif s == :binaries
+        sizes = p.binary_sizes
+
+        bins = Vector{Ptr{UInt8}}(undef, length(sizes))
+        # keep a reference to the underlying binary arrays
+        # as storing the pointer to the array hides the additional
+        # reference from julia's garbage collector
+        bin_arrays = Any[]
+        for (i, s) in enumerate(sizes)
+            if s > 0
+                bin = Vector{UInt8}(undef, s)
+                bins[i] = pointer(bin)
+                push!(bin_arrays, bin)
+            else
+                bins[i] = Base.unsafe_convert(Ptr{UInt8}, C_NULL)
+            end
+        end
+        clGetProgramInfo(p, CL_PROGRAM_BINARIES, sizeof(bins), bins, C_NULL)
+
+        binary_dict = Dict{Device, Array{UInt8}}()
+        bidx = 1
+        for (i, d) in enumerate(p.devices)
+            if sizes[i] > 0
+                binary_dict[d] = bin_arrays[bidx]
+                bidx += 1
+            end
+        end
+        return binary_dict
+    elseif s == :context
+        ctx = Ref{cl_context}()
+        clGetProgramInfo(p, CL_PROGRAM_CONTEXT, sizeof(cl_context), ctx, C_NULL)
+        return Context(ctx[], retain=true)
+    elseif s == :build_status
+        status_dict = Dict{Device, cl_build_status}()
+        for device in p.devices
+            status = Ref{cl_build_status}()
+            clGetProgramBuildInfo(p, device, CL_PROGRAM_BUILD_STATUS, sizeof(cl_build_status), status, C_NULL)
+            status_dict[device] = status[]
+        end
+        return status_dict
+    elseif s == :build_log
+        log_dict = Dict{Device, String}()
+        for device in p.devices
+            size = Ref{Csize_t}()
+            clGetProgramBuildInfo(p, device, CL_PROGRAM_BUILD_LOG, 0, C_NULL, size)
+            log = Vector{Cchar}(undef, size[])
+            clGetProgramBuildInfo(p, device, CL_PROGRAM_BUILD_LOG, size[], log, C_NULL)
+            log_dict[device] = GC.@preserve log unsafe_string(pointer(log))
+        end
+        return log_dict
+    else
+        return getfield(p, s)
+    end
+end
+
 mutable struct Kernel
     const id::cl_kernel
 

src/pocl/pocl.jl

@@ -5,25 +5,31 @@ include("nanoOpenCL.jl")
 import .nanoOpenCL as cl
 
 function platform()
-    for p in cl.platforms()
-        if p.vendor == "The pocl project"
-            return p
+    get!(task_local_storage(), :POCLPlatform) do
+        for p in cl.platforms()
+            if p.vendor == "The pocl project"
+                return p
+            end
         end
-    end
-    return nothing
+        error("POCL not available")
+    end::cl.Platform
 end
 
 function device()
-    p = platform()
-    return cl.default_device(p)
+    get!(task_local_storage(), :POCLDevice) do
+        p = platform()
+        return cl.default_device(p)
+    end::cl.Device
 end
 
-# TODO: Global Context or TLS Context?
 function context()
-    error("TBD")
+    get!(task_local_storage(), :POCLContext) do
+        cl.Context(device())
+    end::cl.Context
 end
 
 using GPUCompiler
+import LLVM
 using Adapt
 
 import SPIRVIntrinsics