Adding Documentation Annotations (#137)

* fixing tests

* adding Documentation annotations

* update to junit5 to accomodate new tests

* going to Java21

* fixing pom and main class

* junit updated to 6.0.0

* code cleanup

* removed not needed java 21 reference

---------

Co-authored-by: Tomas Dvorak <tomas.dvorak@graylog.com>

Author: janheise

.github/workflows/build.yml

@@ -13,7 +13,7 @@ jobs:
 
     strategy:
       matrix:
-        java-version: ["17", "11", "8"]
+        java-version: ["21"]
 
     steps:
       - uses: "actions/checkout@v3"

pom.xml

@@ -73,10 +73,16 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.source>21</maven.compiler.source>
+        <maven.compiler.target>21</maven.compiler.target>
         <maven.site.deploy.skip>true</maven.site.deploy.skip>
         <slf4j.version>2.0.17</slf4j.version>
+        <jakarta.annotation-api.version>3.0.0</jakarta.annotation-api.version>
+        <commons-text.version>1.14.0</commons-text.version>
+        <commons-lang3.version>3.19.0</commons-lang3.version>
+        <commons-csv.version>1.14.1</commons-csv.version>
+        <junit-jupiter.version>6.0.0</junit-jupiter.version>
+        <assertj-core.version>3.27.6</assertj-core.version>
     </properties>
 
     <dependencies>
@@ -91,6 +97,26 @@
             <version>2.14.0</version>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>jakarta.annotation</groupId>
+            <artifactId>jakarta.annotation-api</artifactId>
+            <version>${jakarta.annotation-api.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>${commons-lang3.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>${commons-text.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>${commons-csv.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.google.inject</groupId>
             <artifactId>guice</artifactId>
@@ -110,9 +136,15 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.13.2</version>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>${junit-jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>${assertj-core.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -293,6 +325,10 @@
                 <groupId>org.eluder.coveralls</groupId>
                 <artifactId>coveralls-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
 

src/main/java/com/github/joschi/jadconfig/documentation/ConfigurationBeansSPI.java

@@ -0,0 +1,16 @@
+package com.github.joschi.jadconfig.documentation;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.ServiceLoader;
+
+public class ConfigurationBeansSPI {
+    public static List<Object> loadConfigurationBeans() {
+        return ServiceLoader.load(DocumentedBeansService.class).stream()
+                .map(ServiceLoader.Provider::get)
+                .map(DocumentedBeansService::getDocumentedConfigurationBeans)
+                .flatMap(Collection::stream)
+                .distinct()
+                .toList();
+    }
+}

src/main/java/com/github/joschi/jadconfig/documentation/ConfigurationDocsGenerator.java

@@ -0,0 +1,171 @@
+package com.github.joschi.jadconfig.documentation;
+
+import com.github.joschi.jadconfig.Parameter;
+import com.github.joschi.jadconfig.ReflectionUtils;
+import com.github.joschi.jadconfig.documentation.printers.ConfigFileDocsPrinter;
+import com.github.joschi.jadconfig.documentation.printers.ConfigurationSection;
+import com.github.joschi.jadconfig.documentation.printers.CsvDocsPrinter;
+import com.github.joschi.jadconfig.documentation.printers.DocsPrinter;
+import jakarta.annotation.Nonnull;
+import org.apache.commons.lang3.ClassUtils;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+
+public class ConfigurationDocsGenerator {
+
+    public static void main(String[] args) throws IOException {
+        final ConfigurationDocsGenerator generator = new ConfigurationDocsGenerator();
+        generator.generateDocumentation(parseDocumentationFormat(args), ConfigurationBeansSPI::loadConfigurationBeans);
+    }
+
+    @Nonnull
+    private static DocumentationFormat parseDocumentationFormat(String[] args) {
+        if (args.length != 2) {
+            throw new IllegalArgumentException("This command needs two arguments - a format and file path. For example" + "csv ${project.build.directory}/configuration-documentation.csv");
+        }
+        final String format = args[0].toLowerCase(Locale.ROOT).trim();
+        final String file = args[1];
+        return new DocumentationFormat(format, file);
+    }
+
+    public void generateDocumentation(DocumentationFormat format, Supplier<List<Object>> configurationBeans) throws IOException {
+        final List<ConfigurationSection> sections = detectConfigurationSections(configurationBeans);
+        try (final DocsPrinter writer = createWriter(format)) {
+            writer.write(sections);
+        }
+    }
+
+    private DocsPrinter createWriter(DocumentationFormat format) throws IOException {
+        final FileWriter fileWriter = new FileWriter(format.outputFile(), StandardCharsets.UTF_8);
+        // TODO: if the format list expands, introduce DI and factories for printers
+        return switch (format.format()) {
+            case "csv" -> new CsvDocsPrinter(fileWriter);
+            case "conf" -> new ConfigFileDocsPrinter(fileWriter);
+            default -> throw new IllegalArgumentException("Unsupported format " + format.format());
+        };
+    }
+
+    /**
+     * Collects all configuration options from all available configuration beans.
+     */
+    private List<ConfigurationSection> detectConfigurationSections(Supplier<List<Object>> configurationBeans) {
+        return configurationBeans.get()
+                .stream()
+                .map(ConfigurationDocsGenerator::beanToConfigSections)
+                .sorted(Comparator.comparing(ConfigurationSection::hasPriority).reversed())
+                .toList();
+    }
+
+    private static ConfigurationSection beanToConfigSections(Object configurationBean) {
+
+        String sectionHeading = null;
+        String sectionDescription = null;
+        if (configurationBean.getClass().isAnnotationPresent(DocumentationSection.class)) {
+            final DocumentationSection documentationSection = configurationBean.getClass().getAnnotation(DocumentationSection.class);
+            sectionHeading = documentationSection.heading();
+            sectionDescription = documentationSection.description();
+        }
+
+        final List<ConfigurationEntryWithSection> entries = Arrays.stream(configurationBean.getClass().getDeclaredFields())
+                .filter(f -> f.isAnnotationPresent(Parameter.class))
+                .filter(ConfigurationDocsGenerator::isPublicFacing)
+                .map(f -> toConfigurationEntry(f, configurationBean))
+                .toList();
+
+        final List<ConfigurationEntry> entriesWithoutSection = getEntriesWithoutSection(entries);
+        final List<ConfigurationSection> sortedSections = sectionsFromEntries(entries);
+
+        return new ConfigurationSection(sectionHeading, sectionDescription, sortedSections, entriesWithoutSection);
+    }
+
+    @Nonnull
+    private static List<ConfigurationSection> sectionsFromEntries(List<ConfigurationEntryWithSection> entries) {
+        return entries.stream()
+                .filter(ConfigurationEntryWithSection::hasSection)
+                .collect(groupBySection())
+                .values()
+                .stream()
+                .sorted(sortByPriority())
+                .toList();
+    }
+
+    private static Comparator<ConfigurationSection> sortByPriority() {
+        return Comparator.comparing(ConfigurationSection::hasPriority, Comparator.reverseOrder());
+    }
+
+    private static Collector<ConfigurationEntryWithSection, ?, Map<String, ConfigurationSection>> groupBySection() {
+        return Collectors.groupingBy(ConfigurationEntryWithSection::sectionHeading, Collectors.collectingAndThen(Collectors.toList(), ConfigurationDocsGenerator::toConfigurationSection));
+    }
+
+    private static ConfigurationSection toConfigurationSection(List<ConfigurationEntryWithSection> list) {
+        final ConfigurationEntryWithSection section = list.stream().findFirst().orElseThrow(() -> new IllegalArgumentException("No configuration section found but expected!"));
+        final List<ConfigurationEntry> entries = list.stream().map(ConfigurationEntryWithSection::entry).collect(Collectors.toList());
+        return new ConfigurationSection(section.sectionHeading(), section.sectionDescription(), Collections.emptyList(), entries);
+    }
+
+    @Nonnull
+    private static List<ConfigurationEntry> getEntriesWithoutSection(List<ConfigurationEntryWithSection> entries) {
+        return entries.stream()
+                .filter(e -> !e.hasSection())
+                .map(ConfigurationEntryWithSection::entry)
+                .sorted(Comparator.comparing(ConfigurationEntry::hasPriority, Comparator.reverseOrder()))
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * There are some configuration options not intended for general usage, mainly just for system packages configuration.
+     *
+     * @see Documentation#visible()
+     */
+    private static boolean isPublicFacing(Field f) {
+        return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible();
+    }
+
+    private static ConfigurationEntryWithSection toConfigurationEntry(Field f, Object instance) {
+        final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)).map(Documentation::value).orElse(null);
+        final Parameter parameter = f.getAnnotation(Parameter.class);
+        final String propertyName = parameter.value();
+        final Object defaultValue = getDefaultValue(f, instance);
+        final String type = getType(f);
+        final boolean required = parameter.required();
+
+        final DocumentationSection documentationSection = f.getAnnotation(DocumentationSection.class);
+        String sectionHeading = null;
+        String sectionDescription = null;
+        if (documentationSection != null) {
+            sectionHeading = documentationSection.heading();
+            sectionDescription = documentationSection.description();
+        }
+        final ConfigurationEntry entry = new ConfigurationEntry(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation);
+        return new ConfigurationEntryWithSection(entry, sectionHeading, sectionDescription);
+    }
+
+    private static Object getDefaultValue(Field f, Object instance) {
+        try {
+            return ReflectionUtils.getFieldValue(instance, f);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static String getType(Field f) {
+        if (f.getType().isPrimitive()) { // unify primitive types and wrappers, e.g. int -> Integer
+            return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName();
+        } else {
+            return f.getType().getSimpleName();
+        }
+    }
+}

src/main/java/com/github/joschi/jadconfig/documentation/ConfigurationEntry.java

@@ -0,0 +1,27 @@
+package com.github.joschi.jadconfig.documentation;
+
+import jakarta.annotation.Nullable;
+
+/**
+ * @param configurationBean Class that defines this configuration entry
+ * @param fieldName java field name
+ * @param type Java type name, e.g. String or Integer.
+ * @param configName configuration property name, as written in the config file
+ * @param defaultValue default value declared in the java field, null if not defined
+ * @param required if the configuration property is mandatory (needs default or entry in the config file)
+ * @param documentation textual documentation of this configuration propery
+ */
+public record ConfigurationEntry(
+        Class<?> configurationBean,
+        String fieldName,
+        String type,
+        String configName,
+        @Nullable Object defaultValue,
+        boolean required,
+        String documentation
+) {
+
+    public boolean hasPriority() {
+        return required && defaultValue == null;
+    }
+}

src/main/java/com/github/joschi/jadconfig/documentation/ConfigurationEntryWithSection.java

@@ -0,0 +1,7 @@
+package com.github.joschi.jadconfig.documentation;
+
+public record ConfigurationEntryWithSection(ConfigurationEntry entry, String sectionHeading, String sectionDescription) {
+    public boolean hasSection() {
+        return sectionHeading != null && !sectionHeading.isBlank();
+    }
+}

src/main/java/com/github/joschi/jadconfig/documentation/Documentation.java

@@ -0,0 +1,26 @@
+package com.github.joschi.jadconfig.documentation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * This annotation is used to document configuration files. It allows autogeneration of config documentation.
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface Documentation {
+    /**
+     * We don't want to expose some configuration fields to users. They are internal, required for system packages functionality
+     * or deprecated. Set to false if you want to hide this field from documentation.
+     */
+    boolean visible() default true;
+
+    /**
+     * Description of this configuration property.
+     */
+    String value() default "";
+}

src/main/java/com/github/joschi/jadconfig/documentation/DocumentationFormat.java

@@ -0,0 +1,4 @@
+package com.github.joschi.jadconfig.documentation;
+
+public record DocumentationFormat(String format, String outputFile) {
+}

src/main/java/com/github/joschi/jadconfig/documentation/DocumentationSection.java

@@ -0,0 +1,15 @@
+package com.github.joschi.jadconfig.documentation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.FIELD})
+public @interface DocumentationSection {
+    String heading();
+    String description();
+}

src/main/java/com/github/joschi/jadconfig/documentation/DocumentedBeansService.java

@@ -0,0 +1,7 @@
+package com.github.joschi.jadconfig.documentation;
+
+import java.util.List;
+
+public interface DocumentedBeansService {
+    List<Object> getDocumentedConfigurationBeans();
+}