juliac: log entrypoints and nonstandard typedefs

.gitignore

@@ -15,6 +15,7 @@
 /test/results_*.json
 /test/results_*.dat
 /test/deps
+/test/trimming/Manifest.toml
 
 *.expmap
 *.exe

contrib/juliac/juliac-buildscript.jl

@@ -27,6 +27,62 @@ if Base.JLOptions().trim != 0
     include(joinpath(@__DIR__, "juliac-trim-base.jl"))
 end
 
+const C_friendly_types = Base.IdSet{Any}([    # a few of these are redundant to make it easier to maintain
+    Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float32, Float64, Bool,
+    Cvoid, Cint, Cshort, Clong, Cuint, Cushort, Culong, Cssize_t, Csize_t,
+    Cchar, Cwchar_t, Cstring, Cwstring,
+    RawFD,
+])
+
+function recursively_add_types!(types::Base.IdSet{DataType}, @nospecialize(T::DataType))
+    if T ∉ C_friendly_types
+        if T <: Ptr
+            return recursively_add_types!(types, T.parameters[1])
+        end
+        T.name.module === Core && error("invalid type for juliac: ", T) # exclude internals (they may change)
+        push!(types, T)
+    end
+    for list in (T.parameters, fieldtypes(T))
+        for S in list
+            recursively_add_types!(types, S)
+        end
+    end
+    return types
+end
+
+function write_logfile(io::IO)
+    iotmp = IOBuffer()
+    types = Base.IdSet{DataType}()
+    Base.visit(Core.GlobalMethods) do method
+        if isdefined(method, :ccallable)
+            rt, sig = method.ccallable
+            name = length(method.ccallable) > 2 ? Symbol(method.ccallable[3]) : method.name
+            print(IOContext(iotmp, :print_method_signature_only => true), method)
+            methodstr = String(take!(iotmp))
+            if name !== method.name
+                methodstr = replace(methodstr, String(method.name) => String(name))
+            end
+            println(io, methodstr, "::", rt)
+            for T in sig.parameters[2:end]
+                recursively_add_types!(types, T)
+            end
+        end
+    end
+    println(io)  # blank line separates methods from types
+    for T in types
+        println(io, T)
+        dtfd = Base.DataTypeFieldDesc(T)
+        local fd
+        for i = 1:Base.datatype_nfields(T)
+            fd = dtfd[i]
+            fn = fieldname(T, i)
+            ft = fieldtype(T, i)
+            println(io, "  ", fn, "::", ft, "[", fd.offset, "]")
+        end
+        println(io, fd.offset + fd.size, " bytes")
+    end
+end
+
 # Load user code
 
 import Base.Experimental.entrypoint
@@ -73,6 +129,14 @@ let mod = Base.include(Main, ARGS[1])
     if ARGS[3] == "true"
         ccall(:jl_add_ccallable_entrypoints, Cvoid, ())
     end
+
+    # Export info about entrypoints and structs needed to create header files
+    if length(ARGS) >= 4
+        logfile = ARGS[4]
+        open(logfile, "w") do io
+            write_logfile(io)
+        end
+    end
 end
 
 if Base.JLOptions().trim != 0

test/Makefile

@@ -75,6 +75,7 @@ gcext:
 
 trimming:
 	@$(MAKE) -C $(SRCDIR)/$@ check $(TRIMMING_ARGS)
+	@$(MAKE) -C $(SRCDIR)/$@ clean $(TRIMMING_ARGS)
 
 clangsa:
 	@$(MAKE) -C $(SRCDIR)/$@

test/trimming/Makefile

@@ -33,7 +33,7 @@ JULIAC_BUILDSCRIPT := $(shell $(JULIA) -e 'print(joinpath(Sys.BINDIR, Base.DATAR
 
 #=============================================================================
 
-release: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE)
+release: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) $(BIN)/capplication$(EXE)
 
 $(BIN)/hello-o.a: $(SRCDIR)/hello.jl $(JULIAC_BUILDSCRIPT)
 	$(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-exe true
@@ -42,17 +42,23 @@ $(BIN)/basic_jll-o.a: $(SRCDIR)/basic_jll.jl $(JULIAC_BUILDSCRIPT)
 	$(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()"
 	$(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-exe true
 
+$(BIN)/libsimple-o.a: $(SRCDIR)/libsimple.jl $(JULIAC_BUILDSCRIPT)
+	$(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-lib true $(BIN)/bindinginfo_libsimple.log
+
 $(BIN)/hello$(EXE): $(BIN)/hello-o.a
 	$(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)
 
 $(BIN)/basic_jll$(EXE): $(BIN)/basic_jll-o.a
 	$(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)
 
-check: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE)
+$(BIN)/capplication$(EXE): $(BIN)/capplication.c $(BIN)/libsimple.h $(BIN)/libsimple-o.a
+	$(CC) -I$(BIN) -I$(SRCDIR) -I$(JULIA_LIBDIR) -o $@ $< -Wl,--whole-archive libsimple-o.a -Wl,--no-whole-archive $(LDFLAGS_ADD) $(LDFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS)
+
+check: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) $(BIN)/capplication$(EXE)
 	$(JULIA) --depwarn=error $(SRCDIR)/trimming.jl $<
 
 clean:
-	-rm -f $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) $(BIN)/hello-o.a $(BIN)/basic_jll-o.a
+	-rm -f $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) $(BIN)/hello-o.a $(BIN)/basic_jll-o.a $(BIN)/libsimple-o.a $(BIN)/libsimple.$(SHLIB_EXT) $(BIN)/capplication$(EXE) $(BIN)/bindinginfo_libsimple.log
 
 .PHONY: release clean check
 

test/trimming/capplication.c

@@ -0,0 +1,20 @@
+#include <stdio.h>
+#include "libsimple.h"
+
+int main() {
+    // Example usage of the functions defined in libsimple.h
+    CVectorPair_float_ vecPair;
+    vecPair.from.length = 3;
+    vecPair.from.data = (float[]){1.0f, 2.0f, 3.0f};
+    vecPair.to.length = 3;
+    vecPair.to.data = (float[]){4.0f, 5.0f, 6.0f};
+
+    float sum = copyto_and_sum(vecPair);
+    printf("Sum of copied values: %f\n", sum);
+
+    MyTwoVec list[] = {{1, 2}, {5, 5}, {3, 4}};
+    int32_t count = countsame(list, 3);
+    printf("Count of same vectors: %d\n", count);
+
+    return 0;
+}

test/trimming/libsimple.h

@@ -0,0 +1,22 @@
+#ifndef JULIALIB_LIBSIMPLE_H
+#define JULIALIB_LIBSIMPLE_H
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+typedef struct {
+    int32_t length;
+    float * data;
+} CVector_float_;
+typedef struct {
+    CVector_float_ from;
+    CVector_float_ to;
+} CVectorPair_float_;
+typedef struct {
+    int32_t x;
+    int32_t y;
+} MyTwoVec;
+
+int32_t countsame(MyTwoVec * list, int32_t n);
+float copyto_and_sum(CVectorPair_float_ fromto);
+#endif // JULIALIB_LIBSIMPLE_H

test/trimming/libsimple.jl

@@ -0,0 +1,44 @@
+module SimpleLib
+# Test the logging of entrypoints and types in a C-callable Julia library.
+
+struct CVector{T}
+    length::Cint
+    data::Ptr{T}
+end
+
+struct CVectorPair{T}
+    from::CVector{T}
+    to::CVector{T}
+end
+
+struct MyTwoVec
+    x::Int32
+    y::Int32
+end
+
+Base.@ccallable "copyto_and_sum" function badname(fromto::CVectorPair{Float32})::Float32
+    from, to = unsafe_wrap(Array, fromto.from.data, fromto.from.length), unsafe_wrap(Array, fromto.to.data, fromto.to.length)
+    copyto!(to, from)
+    return sum(to)
+end
+
+Base.@ccallable function countsame(list::Ptr{MyTwoVec}, n::Int32)::Int32
+    list = unsafe_wrap(Array, list, n)
+    count = 0
+    for v in list
+        count += v.x == v.y
+    end
+    return count
+end
+
+export countsame, copyto_and_sum
+
+# FIXME? varargs
+# Base.@ccallable function printints(x::Cint...)::Nothing
+#     for i in 1:length(x)
+#         print(x[i], " ")
+#     end
+#     println()
+# end
+
+end

test/trimming/trimming.jl

@@ -15,4 +15,37 @@ let exe_suffix = splitext(Base.julia_exename())[2]
     @test lines[2] == lines[3]
     @test Base.VersionNumber(lines[2]) ≥ v"1.5.7"
     @test filesize(basic_jll_exe) < filesize(unsafe_string(Base.JLOptions().image_file))/10
+
+    # Test that the shared library can be used in a C application
+    capplication_exe = joinpath(bindir, "capplication" * exe_suffix)
+    lines = split(readchomp(`$capplication_exe`), "\n")
+    @test length(lines) == 2
+    @test lines[1] == "Sum of copied values: 6.000000"
+    @test lines[2] == "Count of same vectors: 1"
+
+    # Test that the logging of entrypoints and types works correctly
+    str = read(joinpath(bindir, "bindinginfo_libsimple.log"), String)
+    @test occursin("copyto_and_sum(fromto::CVectorPair{Float32})::Float32", str)
+    @test occursin(
+        """
+        CVector{Float32}
+          length::Int32[0]
+          data::Ptr{Float32}[8]
+        16 bytes""", str
+    )
+    @test occursin(
+        """
+        CVectorPair{Float32}
+          from::CVector{Float32}[0]
+          to::CVector{Float32}[16]
+        32 bytes""", str
+    )
+    # ensure that there is a blank line between methods and types
+    lines = split(str, '\n'; keepempty=true)
+    nblanks = 0
+    for line in lines
+        nblanks += isempty(line)
+        occursin("length", line) && break
+    end
+    @test nblanks == 1
 end