Docsystem: handle "semantically important docstring" cases (JuliaLang/JuliaLowering.jl#80)

---------

Co-authored-by: Claire Foster <[email protected]>

Author: mlechu

JuliaLowering/src/compat.jl

@@ -275,12 +275,14 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
                                              Expr(:MacroName, a12_esc(a12.value))))
             end
         elseif a1 isa GlobalRef && a1.mod === Core
-            # TODO (maybe): syntax-introduced macrocalls are listed here for
-            # reference.  We probably don't need to convert these.
+            # Syntax-introduced macrocalls are listed here for reference.  We
+            # probably don't need to convert these.
             if a1.name === Symbol("@cmd")
-            elseif a1.name === Symbol("@doc")
+            elseif a1.name === Symbol("@doc") && nargs === 4 # two macro args only
+                # Single-arg @doc is a lookup not corresponding to K"doc"
+                # Revise sometimes calls @doc with three args, but probably shouldn't
                 st_k = K"doc"
-                child_exprs = child_exprs[2:end]
+                child_exprs = child_exprs[2:3]
             elseif a1.name === Symbol("@int128_str")
             elseif a1.name === Symbol("@int128_str")
             elseif a1.name === Symbol("@big_str")

JuliaLowering/src/desugaring.jl

@@ -2857,7 +2857,7 @@ function is_invalid_func_name(ex)
     return is_ccall_or_cglobal(name)
 end
 
-function expand_function_def(ctx, ex, docs, rewrite_call=identity, rewrite_body=identity)
+function expand_function_def(ctx, ex, docs, rewrite_call=identity, rewrite_body=identity; doc_only=false)
     @chk numchildren(ex) in (1,2)
     name = ex[1]
     if numchildren(ex) == 1 && is_identifier_like(name)
@@ -3023,6 +3023,35 @@ function expand_function_def(ctx, ex, docs, rewrite_call=identity, rewrite_body=
         end
     end
 
+    if doc_only
+        # The (doc str (call ...)) form requires method signature lowering, but
+        # does not execute or define any method, so we can't use function_type.
+        # This is a bit of a messy case in the docsystem which we'll hopefully
+        # be able to delete at some point.
+        sig_stmts = SyntaxList(ctx)
+        @assert first_default != 1 && length(arg_types) >= 1
+        last_required = first_default === 0 ? length(arg_types) : first_default - 1
+        for i in last_required:length(arg_types)
+            push!(sig_stmts, @ast(ctx, ex, [K"curly" "Tuple"::K"core" arg_types[2:i]...]))
+        end
+        sig_type = @ast ctx ex [K"where"
+            [K"curly" "Union"::K"core" sig_stmts...] 
+            [K"_typevars" [K"block" typevar_names...] [K"block"]]
+        ]
+        out = @ast ctx docs [K"block"
+            typevar_stmts...
+            [K"call"
+                bind_static_docs!::K"Value"
+                (kind(name) == K"." ? name[1] : ctx.mod::K"Value")
+                name_str::K"Symbol"
+                docs[1]
+                ::K"SourceLocation"(ex)
+                sig_type
+            ]
+        ]
+        return expand_forms_2(ctx, out)
+    end
+
     if !isnothing(return_type)
         ret_var = ssavar(ctx, return_type, "return_type")
         push!(body_stmts, @ast ctx return_type [K"=" ret_var return_type])
@@ -3083,10 +3112,11 @@ function expand_function_def(ctx, ex, docs, rewrite_call=identity, rewrite_body=
     push!(method_stmts,
           method_def_expr(ctx, ex, callex, method_table, main_typevar_names, arg_names,
                           arg_types, body, ret_var))
+
     if !isnothing(docs)
         method_stmts[end] = @ast ctx docs [K"block"
             method_metadata := method_stmts[end]
-            @ast ctx docs [K"call"
+            [K"call"
                 bind_docs!::K"Value"
                 doc_obj
                 docs[1]
@@ -4273,6 +4303,27 @@ function expand_module(ctx, ex::SyntaxTree)
     ]
 end
 
+#-------------------------------------------------------------------------------
+# Expand docstring-annotated expressions
+
+function expand_doc(ctx, ex, docex, mod=ctx.mod)
+    if kind(ex) in (K"Identifier", K".")
+        expand_forms_2(ctx, @ast ctx docex [K"call"
+            bind_static_docs!::K"Value"
+            (kind(ex) === K"." ? ex[1] : ctx.mod::K"Value")
+            (kind(ex) === K"." ? ex[2] : ex).name_val::K"Symbol"
+            docex[1]
+            ::K"SourceLocation"(ex)
+            Union{}::K"Value"
+        ])
+    elseif is_eventually_call(ex)
+        expand_function_def(ctx, @ast(ctx, ex, [K"function" ex [K"block"]]),
+                            docex; doc_only=true)
+    else
+        expand_forms_2(ctx, ex, docex)
+    end
+end
+
 #-------------------------------------------------------------------------------
 # Desugaring's "big switch": expansion of some simple forms; dispatch to other
 # expansion functions for the rest.
@@ -4339,7 +4390,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
         expand_forms_2(ctx, expand_compare_chain(ctx, ex))
     elseif k == K"doc"
         @chk numchildren(ex) == 2
-        sig = expand_forms_2(ctx, ex[2], ex)
+        expand_doc(ctx, ex[2], ex)
     elseif k == K"for"
         expand_forms_2(ctx, expand_for(ctx, ex))
     elseif k == K"comprehension"

JuliaLowering/src/runtime.jl

@@ -315,6 +315,21 @@ function bind_docs!(type::Type, docstr, lineno::LineNumberNode; field_docs=Core.
     Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), Union{})
 end
 
+"""
+Called in the unfortunate cases (K"call", K".", K"Identifier") where docstrings
+change the semantics of the expressions they annotate, no longer requiring the
+expression to execute.
+"""
+function bind_static_docs!(mod::Module, name::Symbol, docstr, lnn::LineNumberNode, sigtypes::Type)
+    metadata = Dict{Symbol, Any}(
+        :linenumber => lnn.line,
+        :module => mod,
+        :path => something(lnn.file, "none"),
+    )
+    bind = Base.Docs.Binding(mod, name)
+    Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), sigtypes)
+end
+
 #--------------------------------------------------
 # Runtime support infrastructure for `@generated`
 

JuliaLowering/test/misc.jl

@@ -104,4 +104,59 @@ end
     @test lower_str(Main, "x + y").args[1].has_image_globalref === true
 end
 
+@testset "docstrings: doc-only expressions" begin
+    local jeval(mod, str) = JuliaLowering.include_string(mod, str; expr_compat_mode=true)
+    jeval(test_mod, "function fun_exists(x); x; end")
+    jeval(test_mod, "module M end; module M2 end")
+    # TODO: return values are to be determined, currently Base.Docs.Binding for
+    # both lowering implementations.  We can't return the value of the
+    # expression in these special cases.
+    jeval(test_mod, "\"docstr1\" sym_noexist")
+    jeval(test_mod, "\"docstr2\" fun_noexist()")
+    jeval(test_mod, "\"docstr3\" fun_exists(sym_noexist)")
+    jeval(test_mod, "\"docstr4\" M.sym_noexist")
+    jeval(test_mod, "\"docstr5\" M.fun_noexist()")
+    jeval(test_mod, "\"docstr6\" M.fun_exists(sym_noexist)")
+    @test jeval(test_mod, "@doc sym_noexist")               |> string === "docstr1\n"
+    @test jeval(test_mod, "@doc fun_noexist()")             |> string === "docstr2\n"
+    @test jeval(test_mod, "@doc fun_exists(sym_noexist)")   |> string === "docstr3\n"
+    @test jeval(test_mod, "@doc M.sym_noexist")             |> string === "docstr4\n"
+    @test jeval(test_mod, "@doc M.fun_noexist()")           |> string === "docstr5\n"
+    @test jeval(test_mod, "@doc M.fun_exists(sym_noexist)") |> string === "docstr6\n"
+    @test jeval(test_mod.M, "@doc M.sym_noexist")             |> string === "docstr4\n"
+    @test jeval(test_mod.M, "@doc M.fun_noexist()")           |> string === "docstr5\n"
+    @test jeval(test_mod.M, "@doc M.fun_exists(sym_noexist)") |> string === "docstr6\n"
+
+    jeval(test_mod.M2, "\"docstr7\" M2.M2.sym_noexist")
+    jeval(test_mod.M2, "\"docstr8\" M2.M2.fun_noexist()")
+    jeval(test_mod.M2, "\"docstr9\" M2.M2.fun_exists(sym_noexist)")
+    @test jeval(test_mod, "@doc M2.M2.sym_noexist")             |> string === "docstr7\n"
+    @test jeval(test_mod, "@doc M2.M2.fun_noexist()")           |> string === "docstr8\n"
+    @test jeval(test_mod, "@doc M2.M2.fun_exists(sym_noexist)") |> string === "docstr9\n"
+    @test jeval(test_mod.M2, "@doc M2.M2.sym_noexist")             |> string === "docstr7\n"
+    @test jeval(test_mod.M2, "@doc M2.M2.fun_noexist()")           |> string === "docstr8\n"
+    @test jeval(test_mod.M2, "@doc M2.M2.fun_exists(sym_noexist)") |> string === "docstr9\n"
+
+    # Try with signatures and type variables
+    jeval(test_mod, "abstract type T_exists end")
+
+    jeval(test_mod, "\"docstr10\" f10(x::Int, y, z::T_exists)")
+    d = jeval(test_mod, "@doc f10")
+    @test d |> string === "docstr10\n"
+    # TODO: Is there a better way of accessing this? Feel free to change tests
+    # if docsystem storage changes.
+    @test d.meta[:results][1].data[:typesig] === Tuple{Int, Any, test_mod.T_exists}
+
+    jeval(test_mod, "\"docstr11\" f11(x::T_exists, y::U, z::T) where {T, U<:Number}")
+    d = jeval(test_mod, "@doc f11")
+    @test d |> string === "docstr11\n"
+    @test d.meta[:results][1].data[:typesig] === Tuple{test_mod.T_exists, U, T} where {T, U<:Number}
+
+    jeval(test_mod, "\"docstr12\" f12(x::Int, y::U, z::T=1) where {T, U<:Number}")
+    d = jeval(test_mod, "@doc f12")
+    @test d |> string === "docstr12\n"
+    @test d.meta[:results][1].data[:typesig] === Union{Tuple{Int64, U, T}, Tuple{Int64, U}} where {T, U<:Number}
+
+end
+
 end