Cleanup

Author: swallez

docs/docset.yml

@@ -13,3 +13,5 @@ subs:
   es3:   "Elasticsearch Serverless"
   es:   "Elasticsearch"
   version: "9.0.0"
+  # Not used by docs-builder, but present in `% :::{include-bloc}` preprocessing directives (see IncludeExpander.java)
+  doc-tests-src: "../../java-client/src/test/java/co/elastic/clients/documentation"

tools/src/main/java/co/elastic/clients/tools/docs/IncludeExpander.java

@@ -30,17 +30,33 @@
 import java.util.Map;
 import java.util.Objects;
 
+/**
+ * Processes the {@code .md} files in a directory expanding code blocks following a custom directive
+ * (expressed as MyST comment):
+ * <pre>
+ *     % :::{include-code} src={{doc-tests-src}}/path/to/SomeCode.java tag=some-marker
+ *     ```java
+ *       this is replaced by the section of SomeCode.java delimited
+ *       with `tag::some-marker` and `end::some-marker`
+ *     ```
+ * </pre>
+ *
+ * Note: the MyST format has the {@code {literalinclude}} directive, which is currently only partially supported
+ * by docs-builder (the whole file is included, ignoring {@code start-after} and {@code end-before} parameters).
+ * <p>
+ * See <a href="https://mystmd.org/guide/code#docs-literalinclude">MyST docs</a>
+ */
 public class IncludeExpander {
 
     public static void main(String[] args) throws IOException {
 
-
         File dir = new File(args.length == 0 ? "docs/reference" : args[0]);
 
         if (!dir.isDirectory()) {
             throw new IllegalArgumentException(dir.getAbsolutePath() + " is not a directory");
         }
 
+        // FIXME: we should read this from `docset.yml` in the `dir` directory.
         processDirectory(dir, Map.of("doc-tests-src", "java-client/src/test/java/co/elastic/clients/documentation"));
     }
 
@@ -79,8 +95,8 @@ public static void processFile(File file, Map<String, String> subst) throws IOEx
 
     enum State {
         NORMAL_TEXT,
-        INCLUDE_CODE,
-        CODE_BLOCK,
+        INCLUDE_CODE_DIRECTIVE,
+        REPLACED_CODE_BLOCK,
     }
 
     public static void fail(String message, String path, LineNumberReader reader, Throwable e) {
@@ -107,28 +123,28 @@ public static String expandText(String input, Map<String, String> subst, String
                     if (line.startsWith("% :::{include-code}")) {
                         output.append(line).append("\n");
                         includeCodeLine = line;
-                        state = State.INCLUDE_CODE;
+                        state = State.INCLUDE_CODE_DIRECTIVE;
 
                     } else {
                         // Regular text line
                         output.append(line).append("\n");
                     }
                 }
 
-                case INCLUDE_CODE -> {
+                case INCLUDE_CODE_DIRECTIVE -> {
                     if (!line.startsWith("```")) {
                         fail("The '% :::{include-code}' should be followed by a code block", path, reader, null);
                     }
                     output.append(line).append("\n");
-                    state = State.CODE_BLOCK;
+                    state = State.REPLACED_CODE_BLOCK;
                     try {
                         expandIncludeCodeDirective(includeCodeLine, subst, output);
                     } catch (Exception e) {
                         fail("Failed to expand include directive", path, reader, e);
                     }
                 }
 
-                case CODE_BLOCK -> {
+                case REPLACED_CODE_BLOCK -> {
                     // Skip existing code until we reach the end
                     if (line.startsWith("```")) {
                         output.append(line).append("\n");
@@ -156,7 +172,7 @@ public static void expandIncludeCodeDirective(String command, Map<String, String
         var src = Objects.requireNonNull(args.get("src"), "Missing 'src' attribute");
         var tag = Objects.requireNonNull(args.get("tag"), "Missing 'tag' attribute");
 
-        // Brute force replacement of placeholders
+        // Brute force replacement of placeholders. We can do better.
         for (var kv: subst.entrySet()) {
             src = src.replace("{{" + kv.getKey() + "}}", kv.getValue());
         }