feat: add test to detect public names without a docstring (#313)

Co-authored-by: Max Horn <[email protected]>
Co-authored-by: Lars Göttgens <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Version [v0.8.12] - unreleased
+
+### Changed
+
+- Add `test_undocumented_names` to verify that all public symbols have docstrings (not including the module itself). This test is not enabled by default in `test_all`. ([#313])
+
 ## Version [v0.8.11] - 2025-02-06
 
 ### Changed

docs/make.jl

@@ -29,6 +29,7 @@ makedocs(;
             "deps_compat.md",
             "piracies.md",
             "persistent_tasks.md",
+            "undocumented_names.md",
         ],
         "release-notes.md",
     ],

docs/src/undocumented_names.md

@@ -0,0 +1,7 @@
+# Undocumented names
+
+## [Test function](@id test_undocumented_names)
+
+```@docs
+Aqua.test_undocumented_names
+```

src/Aqua.jl

@@ -1,6 +1,6 @@
 module Aqua
 
-using Base: PkgId, UUID
+using Base: Docs, PkgId, UUID
 using Pkg: Pkg, TOML, PackageSpec
 using Test
 
@@ -23,6 +23,7 @@ include("stale_deps.jl")
 include("deps_compat.jl")
 include("piracies.jl")
 include("persistent_tasks.jl")
+include("undocumented_names.jl")
 
 """
     test_all(testtarget::Module)
@@ -37,6 +38,7 @@ Run the following tests:
 * [`test_deps_compat(testtarget)`](@ref test_deps_compat)
 * [`test_piracies(testtarget)`](@ref test_piracies)
 * [`test_persistent_tasks(testtarget)`](@ref test_persistent_tasks)
+* [`test_undocumented_names(testtarget)`](@ref test_undocumented_names)
 
 The keyword argument `\$x` (e.g., `ambiguities`) can be used to
 control whether or not to run `test_\$x` (e.g., `test_ambiguities`).
@@ -52,6 +54,7 @@ passed to `\$x` to specify the keyword arguments for `test_\$x`.
 - `deps_compat = true`
 - `piracies = true`
 - `persistent_tasks = true`
+- `undocumented_names = false`
 """
 function test_all(
     testtarget::Module;
@@ -63,6 +66,7 @@ function test_all(
     deps_compat = true,
     piracies = true,
     persistent_tasks = true,
+    undocumented_names = false,
 )
     if ambiguities !== false
         @testset "Method ambiguity" begin
@@ -105,6 +109,13 @@ function test_all(
             test_persistent_tasks(testtarget; askwargs(persistent_tasks)...)
         end
     end
+    @testset "Undocumented names" begin
+        if undocumented_names !== false
+            isempty(askwargs(undocumented_names)) ||
+                error("Keyword arguments not supported")
+            test_undocumented_names(testtarget; askwargs(undocumented_names)...)
+        end
+    end
 end
 
 end # module

src/undocumented_names.jl

@@ -0,0 +1,45 @@
+"""
+    test_undocumented_names(module::Module)
+
+Test that all public names in `module` and its recursive submodules have a docstring (not including `module` itself).
+
+!!! tip
+    On all Julia versions, public names include the exported names.
+    On Julia versions >= 1.11, public names also include the names annotated with the `public` keyword.
+
+!!! warning
+    When running this Aqua test in Julia versions before 1.11, it does nothing.
+    Thus if you use continuous integration tests, make sure those are configured
+    to use Julia >= 1.11 in order to benefit from this test.
+
+# Keyword Arguments
+- `broken::Bool = false`: If true, it uses `@test_broken` instead of
+  `@test` and shortens the error message.
+"""
+function test_undocumented_names(m::Module; broken::Bool = false)
+    @static if VERSION >= v"1.11"
+        # exclude the module name itself because it has the README as auto-generated docstring (https://github.com/JuliaLang/julia/pull/39093)
+        undocumented_names = Symbol[]
+        walkmodules(m) do x
+            append!(undocumented_names, Docs.undocumented_names(x))
+        end
+        undocumented_names = filter(n -> n != nameof(m), undocumented_names)
+        if broken
+            @test_broken isempty(undocumented_names)
+        else
+            @test isempty(undocumented_names)
+        end
+    else
+        undocumented_names = Symbol[]
+    end
+    if !isempty(undocumented_names)
+        printstyled(
+            stderr,
+            "Undocumented names detected:\n";
+            bold = true,
+            color = Base.error_color(),
+        )
+        !broken && show(stderr, MIME"text/plain"(), undocumented_names)
+        println(stderr)
+    end
+end

test/pkgs/PkgWithUndocumentedNames.jl

@@ -0,0 +1,20 @@
+module PkgWithUndocumentedNames
+
+"""
+    documented_function
+"""
+function documented_function end
+
+function undocumented_function end
+
+"""
+    DocumentedStruct
+"""
+struct DocumentedStruct end
+
+struct UndocumentedStruct end
+
+export documented_function, DocumentedStruct
+export undocumented_function, UndocumentedStruct
+
+end  # module

test/pkgs/PkgWithUndocumentedNamesInSubmodule.jl

@@ -0,0 +1,14 @@
+module PkgWithUndocumentedNamesInSubmodule
+
+"""
+    DocumentedStruct
+"""
+struct DocumentedStruct end
+
+module SubModule
+
+struct UndocumentedStruct end
+
+end
+
+end  # module

test/pkgs/PkgWithoutUndocumentedNames.jl

@@ -0,0 +1,18 @@
+"""
+    PkgWithoutUndocumentedNames
+"""
+module PkgWithoutUndocumentedNames
+
+"""
+    documented_function
+"""
+function documented_function end
+
+"""
+    DocumentedStruct
+"""
+struct DocumentedStruct end
+
+export documented_function, DocumentedStruct
+
+end  # module

test/test_undocumented_names.jl

@@ -0,0 +1,58 @@
+module TestUndocumentedNames
+
+include("preamble.jl")
+
+import PkgWithUndocumentedNames
+import PkgWithUndocumentedNamesInSubmodule
+import PkgWithoutUndocumentedNames
+
+# Pass
+results = @testtestset begin
+    Aqua.test_undocumented_names(PkgWithoutUndocumentedNames)
+end
+if VERSION >= v"1.11"
+    @test length(results) == 1
+    @test results[1] isa Test.Pass
+else
+    @test length(results) == 0
+end
+
+# Fail
+println("### Expected output START ###")
+results = @testtestset begin
+    Aqua.test_undocumented_names(PkgWithUndocumentedNames)
+end
+println("### Expected output END ###")
+if VERSION >= v"1.11"
+    @test length(results) == 1
+    @test results[1] isa Test.Fail
+else
+    @test length(results) == 0
+end
+
+println("### Expected output START ###")
+results = @testtestset begin
+    Aqua.test_undocumented_names(PkgWithUndocumentedNamesInSubmodule)
+end
+println("### Expected output END ###")
+if VERSION >= v"1.11"
+    @test length(results) == 1
+    @test results[1] isa Test.Fail
+else
+    @test length(results) == 0
+end
+
+# Broken
+println("### Expected output START ###")
+results = @testtestset begin
+    Aqua.test_undocumented_names(PkgWithUndocumentedNames; broken = true)
+end
+println("### Expected output END ###")
+if VERSION >= v"1.11"
+    @test length(results) == 1
+    @test results[1] isa Test.Broken
+else
+    @test length(results) == 0
+end
+
+end  # module