Export ABI metadata in JSON format

With luck, this makes this information easier to consume downstream (and
to expand on in the future).

Also fixes a couple of obscure bugs that would cause mis-behavior in the
presence of recursive types, or which could unintentionally mangle types
in the event of sub-string collisions.

Author: topolarity

contrib/juliac/juliac-buildscript.jl

@@ -35,8 +35,11 @@ const C_friendly_types = Base.IdSet{Any}([    # a few of these are redundant to
 ])
 
 function recursively_add_types!(types::Base.IdSet{DataType}, @nospecialize(T::DataType))
-    while T <: Ptr
+    T in types && return types
+    while T.name === Ptr.body.name
+        push!(types, T)
         T = T.parameters[1] # unwrap Ptr{...}
+        T in types && return types
     end
     if T.name.module === Core && T ∉ C_friendly_types
         error("invalid type for juliac: ", T) # exclude internals (they may change)
@@ -50,37 +53,198 @@ function recursively_add_types!(types::Base.IdSet{DataType}, @nospecialize(T::Da
     return types
 end
 
+struct TypeEmitter
+    io::IO
+    type_ids::IdDict{Any,Int}
+end
+
+function escape_string_json(s::AbstractString)
+    iob = IOBuffer()
+    print(iob, '"')
+    for c in s
+        if c == '"'
+            print(iob, "\\\"")
+        elseif c == '\\'
+            print(iob, "\\\\")
+        elseif c == '\b'
+            print(iob, "\\b")
+        elseif c == '\f'
+            print(iob, "\\f")
+        elseif c == '\n'
+            print(iob, "\\n")
+        elseif c == '\r'
+            print(iob, "\\r")
+        elseif c == '\t'
+            print(iob, "\\t")
+        elseif '\x00' <= c <= '\x1f'
+            print(iob, "\\u", lpad(string(UInt16(c), base=16), 4, '0'))
+        else
+            @assert isvalid(c) "invalid unicode character"
+            print(iob, c)
+        end
+    end
+    print(iob, '"')
+    return String(take!(iob))
+end
+
+function type_name_json(@nospecialize(dt::DataType))
+    return escape_string_json(repr(dt))
+end
+
+function field_name_json(@nospecialize(dt::DataType), field::Int)
+    name = String(fieldname(dt, field))
+    return escape_string_json(name)
+end
+
+function emit_pointer_info!(ctx::TypeEmitter, @nospecialize(dt::DataType); indent::Int = 0)
+    pointee_type_id = ctx.type_ids[dt.parameters[1]]
+    let indented_println(args...) = println(ctx.io, " " ^ indent, args...)
+        indented_println("{")
+        indented_println("  \"id\": ", ctx.type_ids[dt], ",")
+        indented_println("  \"kind\": \"pointer\",")
+        indented_println("  \"name\": ", type_name_json(dt), ",")
+        indented_println("  \"pointee\": ", pointee_type_id)
+        print(ctx.io, " " ^ indent, "}")
+    end
+end
+
+function emit_field_info!(ctx::TypeEmitter, @nospecialize(dt::DataType), field::Int; indent::Int = 0)
+    desc = Base.DataTypeFieldDesc(dt)[field]
+    type_id = ctx.type_ids[fieldtype(dt, field)]
+    let indented_println(args...) = println(ctx.io, " " ^ indent, args...)
+        indented_println("{")
+        indented_println("  \"name\": ", field_name_json(dt, field), ",")
+        indented_println("  \"offset\": ", desc.offset, ",")
+        indented_println("  \"type\": ", type_id)
+        print(ctx.io, " " ^ indent, "}")
+    end
+end
+
+function emit_struct_info!(ctx::TypeEmitter, @nospecialize(dt::DataType); indent::Int = 0)
+    type_id = ctx.type_ids[dt]
+    let indented_println(args...) = println(ctx.io, " " ^ indent, args...)
+        indented_println("{")
+        indented_println("  \"id\": ", type_id, ",")
+        indented_println("  \"kind\": \"struct\",")
+        indented_println("  \"name\": ", type_name_json(dt), ",")
+        indented_println("  \"size\": ", Core.sizeof(dt), ",")
+        indented_println("  \"alignment\": ", Base.datatype_alignment(dt), ",")
+        indented_println("  \"fields\": [")
+        for i = 1:Base.datatype_nfields(dt)
+            emit_field_info!(ctx, dt, i; indent = indent + 4)
+            println(ctx.io, i == Base.datatype_nfields(dt) ? "" : ",")
+        end
+        indented_println("  ]")
+        print(ctx.io, " " ^ indent, "}")
+    end
+end
+
+function emit_primitive_type!(ctx::TypeEmitter, @nospecialize(dt::DataType); indent::Int = 0)
+    type_id = ctx.type_ids[dt]
+    let indented_println(args...) = println(ctx.io, " " ^ indent, args...)
+        indented_println("{")
+        indented_println("  \"id\": ", type_id, ",")
+        indented_println("  \"kind\": \"primitive\",")
+        indented_println("  \"name\": ", type_name_json(dt), ",")
+        indented_println("  \"size\": ", Core.sizeof(dt), ",")
+        indented_println("  \"alignment\": ", Base.datatype_alignment(dt))
+        print(ctx.io, " " ^ indent, "}")
+    end
+end
+
+function emit_type_info!(ctx::TypeEmitter, @nospecialize(dt::DataType); indent::Int = 0)
+    if dt.name === Ptr.body.name
+        emit_pointer_info!(ctx, dt; indent)
+    elseif Base.isprimitivetype(dt)
+        emit_primitive_type!(ctx, dt; indent)
+    else
+        emit_struct_info!(ctx, dt; indent)
+    end
+end
+
+function emit_method_info!(ctx::TypeEmitter, method::Core.Method; indent::Int = 0)
+    (rt, sig) = method.ccallable
+    (name, symbol) = let
+        symbol = length(method.ccallable) > 2 ? Symbol(method.ccallable[3]) : method.name
+        iob = IOBuffer()
+        print(IOContext(iob, :print_method_signature_only => true), method)
+        str = String(take!(iob))
+        if symbol !== method.name && startswith(str, String(method.name))
+            # Make a best-effort attempt to use the exported name
+            #
+            # Note: the `startswith` check is to make sure we support 'functor's in arg0,
+            # which Base.@ccallable supports as long as they are singletons.
+            str = replace(str, String(method.name) => String(symbol); count = 1)
+        end
+        (str, String(symbol))
+    end
+
+    argnames = String.(Base.method_argnames(method))
+    let indented_println(args...) = println(ctx.io, " " ^ indent, args...)
+        indented_println("{")
+        indented_println("  \"symbol\": ", escape_string_json(symbol), ",")
+        indented_println("  \"name\": ", escape_string_json(name), ",")
+        indented_println("  \"arguments\": [")
+        for i in 2:length(sig.parameters)
+            indented_println("    {")
+            indented_println("      \"name\": ", escape_string_json(argnames[i]), ",")
+            indented_println("      \"type\": ", ctx.type_ids[sig.parameters[i]])
+            indented_println("    }", i == length(sig.parameters) ? "" : ",")
+        end
+        indented_println("  ],")
+        indented_println("  \"returns\": {")
+        indented_println("    \"type\": ", ctx.type_ids[rt])
+        indented_println("  }")
+        print(ctx.io, " " ^ indent, "}")
+    end
+end
+
+function emit_abi_info!(ctx::TypeEmitter, exported::Vector{Core.Method}, types::IdSet{DataType})
+    println(ctx.io, "{")
+
+    # assign an ID to each type, so that we can refer to them
+    for (i, T) in enumerate(types)
+        ctx.type_ids[T] = i
+    end
+
+    # print exported functions
+    println(ctx.io, "  \"functions\": [")
+    for (i, method) in enumerate(exported)
+        emit_method_info!(ctx, method; indent = 4)
+        println(ctx.io, i == length(exported) ? "" : ",")
+    end
+    println(ctx.io, "  ],")
+
+    # print type / structure information
+    println(ctx.io, "  \"types\": [")
+    for (i, T) in enumerate(types)
+        emit_type_info!(ctx, T; indent = 4)
+        println(ctx.io, i == length(types) ? "" : ",")
+    end
+    println(ctx.io, "  ]")
+
+    println(ctx.io, "}")
+end
+
 function write_logfile(io::IO)
-    iotmp = IOBuffer()
     types = Base.IdSet{DataType}()
+
+    # discover all exported methods + any types they reference
+    exported = Core.Method[]
     Base.visit(Core.GlobalMethods) do method
         if isdefined(method, :ccallable)
-            rt, sig = method.ccallable
-            name = length(method.ccallable) > 2 ? Symbol(method.ccallable[3]) : method.name
-            print(IOContext(iotmp, :print_method_signature_only => true), method)
-            methodstr = String(take!(iotmp))
-            if name !== method.name
-                methodstr = replace(methodstr, String(method.name) => String(name))
-            end
-            println(io, methodstr, "::", rt)
+            push!(exported, method)
+            (rt, sig) = method.ccallable
             for T in sig.parameters[2:end]
                 recursively_add_types!(types, T)
             end
+            recursively_add_types!(types, rt)
         end
     end
-    println(io)  # blank line separates methods from types
-    for T in types
-        println(io, T)
-        dtfd = Base.DataTypeFieldDesc(T)
-        local fd
-        for i = 1:Base.datatype_nfields(T)
-            fd = dtfd[i]
-            fn = fieldname(T, i)
-            ft = fieldtype(T, i)
-            println(io, "  ", fn, "::", ft, "[", fd.offset, "]")
-        end
-        println(io, fd.offset + fd.size, " bytes")
-    end
+
+    # print the discovered ABI info
+    ctx = TypeEmitter(io, IdDict{Any,Int}())
+    emit_abi_info!(ctx, exported, types)
 end
 
 # Load user code

test/trimming/libsimple.jl

@@ -16,6 +16,20 @@ struct MyTwoVec
     y::Int32
 end
 
+struct CTree{T}
+    # test that recursive datatypes work as expected
+    children::CVector{CTree{T}}
+end
+
+Base.@ccallable "tree_size" function size(tree::CTree{Float64})::Int64
+    children = unsafe_wrap(Array, tree.children.data, tree.children.length)
+    # Return the size of this sub-tree
+    return sum(Int64[
+        size(child)
+        for child in children
+    ]; init=1)
+end
+
 Base.@ccallable "copyto_and_sum" function badname(fromto::CVectorPair{Float32})::Float32
     from, to = unsafe_wrap(Array, fromto.from.data, fromto.from.length), unsafe_wrap(Array, fromto.to.data, fromto.to.length)
     copyto!(to, from)