Merge pull request #7 from smithy-lang/sphinx-autobuild

Automatically build sphinx projects

Author: JordonPhillips

smithy-docgen-core/src/main/java/software/amazon/smithy/docgen/core/DocgenUtils.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.docgen.core;
+
+import static java.lang.String.format;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+import software.amazon.smithy.codegen.core.CodegenException;
+import software.amazon.smithy.utils.SmithyUnstableApi;
+
+/**
+ * Provides various utility methods.
+ */
+@SmithyUnstableApi
+public final class DocgenUtils {
+
+    private static final Logger LOGGER = Logger.getLogger(DocgenUtils.class.getName());
+
+    private DocgenUtils() {}
+
+    /**
+     * Executes a given shell command in a given directory.
+     *
+     * @param command The string command to execute, e.g. "sphinx-build".
+     * @param directory The directory to run the command in.
+     * @return Returns the console output of the command.
+     */
+    public static String runCommand(String command, Path directory) {
+        String[] finalizedCommand;
+        if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+            finalizedCommand = new String[]{"cmd.exe", "/c", command};
+        } else {
+            finalizedCommand = new String[]{"sh", "-c", command};
+        }
+
+        ProcessBuilder processBuilder = new ProcessBuilder(finalizedCommand)
+                .redirectErrorStream(true)
+                .directory(directory.toFile());
+
+        try {
+            Process process = processBuilder.start();
+            List<String> output = new ArrayList<>();
+
+            // Capture output for reporting.
+            try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(
+                process.getInputStream(), Charset.defaultCharset()))) {
+                String line;
+                while ((line = bufferedReader.readLine()) != null) {
+                    LOGGER.finest(line);
+                    output.add(line);
+                }
+            }
+
+            process.waitFor();
+            process.destroy();
+
+            String joinedOutput = String.join(System.lineSeparator(), output);
+            if (process.exitValue() != 0) {
+                throw new CodegenException(format(
+                    "Command `%s` failed with output:%n%n%s", command, joinedOutput));
+            }
+            return joinedOutput;
+        } catch (InterruptedException | IOException e) {
+            throw new CodegenException(e);
+        }
+    }
+
+    /**
+     * Replaces all newline characters in a string with the system line separator.
+     * @param input The string to normalize
+     * @return A string with system-appropriate newlines.
+     */
+    public static String normalizeNewlines(String input) {
+        return input.replaceAll("\r?\n", System.lineSeparator());
+    }
+}

smithy-docgen-core/src/main/java/software/amazon/smithy/docgen/core/integrations/SphinxIntegration.java

@@ -5,29 +5,68 @@
 
 package software.amazon.smithy.docgen.core.integrations;
 
+import static java.lang.String.format;
+import static software.amazon.smithy.docgen.core.DocgenUtils.normalizeNewlines;
+import static software.amazon.smithy.docgen.core.DocgenUtils.runCommand;
+
 import java.util.List;
 import java.util.Set;
 import java.util.logging.Logger;
+import software.amazon.smithy.codegen.core.CodegenException;
 import software.amazon.smithy.docgen.core.DocFormat;
 import software.amazon.smithy.docgen.core.DocGenerationContext;
 import software.amazon.smithy.docgen.core.DocIntegration;
 import software.amazon.smithy.docgen.core.DocSettings;
 import software.amazon.smithy.docgen.core.sections.sphinx.ConfSection;
 import software.amazon.smithy.docgen.core.sections.sphinx.MakefileSection;
+import software.amazon.smithy.docgen.core.sections.sphinx.RequirementsSection;
 import software.amazon.smithy.docgen.core.sections.sphinx.WindowsMakeSection;
 import software.amazon.smithy.docgen.core.writers.SphinxMarkdownWriter;
 import software.amazon.smithy.model.shapes.ServiceShape;
 import software.amazon.smithy.utils.SmithyInternalApi;
 
 /**
  * Adds Sphinx project scaffolding for compatible formats.
+ *
+ * <p>This integration runs in low priority to allow other integrations to generate
+ * files that will be picked up by sphinx-build. To have an integration reliably run
+ * after this, override {@link DocIntegration#runAfter} with the output of
+ * {@link SphinxIntegration#name} in the list. Similarly, to guarantee an integration
+ * is run before this, override {@link DocIntegration#runBefore} with the same argument.
+ *
+ * <p>To customize the project files generated by this integration, you can make use
+ * of {@link DocIntegration#interceptors} to intercept and modify the files before
+ * they're written. The following named code sections are used:
+ *
+ * <ul>
+ *     <li>{@link ConfSection}: Creates the {@code conf.py}
+ *     <li>{@link MakefileSection}: Creates the {@code Makefile} build script for unix.
+ *     <li>{@link WindowsMakeSection}: Creates the {@code make.bat} build script for
+ *     Windows.
+ *     <li>{@link RequirementsSection}: Creates the {@code requirements.txt} used to
+ *     build the docs. Any dependencies here will be installed into the environment
+ *     used to run {@code sphinx-build}.
+ * </ul>
  */
 @SmithyInternalApi
 public final class SphinxIntegration implements DocIntegration {
     private static final String MARKDOWN_FORMAT = "sphinx-markdown";
     private static final Set<String> FORMATS = Set.of(MARKDOWN_FORMAT);
     private static final Logger LOGGER = Logger.getLogger(SphinxIntegration.class.getName());
 
+    // The default requirements needed to build the docs.
+    private static final List<String> REQUIREMENTS = List.of(
+        "Sphinx==7.2.6",
+        "myst-parser==2.0.0",
+        "linkify-it-py==2.0.2"
+    );
+
+    @Override
+    public byte priority() {
+        // Run at the end so that any integration-generated changes can happen.
+        return -128;
+    }
+
     @Override
     public List<DocFormat> docFormats(DocSettings settings) {
         return List.of(
@@ -38,16 +77,26 @@ public List<DocFormat> docFormats(DocSettings settings) {
     @Override
     public void customize(DocGenerationContext context) {
         if (!FORMATS.contains(context.docFormat().name())) {
-            LOGGER.finest(String.format(
+            LOGGER.finest(format(
                 "Format %s is not a Sphinx-compatible format, skipping Sphinx project setup.",
                 context.docFormat().name()
             ));
             return;
         }
         // TODO: add some way to disable project file generation
-        LOGGER.finest("Generating Sphinx project files.");
+        LOGGER.info("Generating Sphinx project files.");
+        writeRequirements(context);
         writeConf(context);
         writeMakefile(context);
+        runSphinx(context);
+    }
+
+    private void writeRequirements(DocGenerationContext context) {
+        context.writerDelegator().useFileWriter("requirements.txt", writer -> {
+            writer.pushState(new RequirementsSection(context, REQUIREMENTS));
+            REQUIREMENTS.forEach(writer::write);
+            writer.popState();
+        });
     }
 
     private void writeConf(DocGenerationContext context) {
@@ -155,4 +204,85 @@ private void writeMakefile(DocGenerationContext context) {
             writer.popState();
         });
     }
+
+    private void runSphinx(DocGenerationContext context) {
+        var baseDir = context.fileManifest().getBaseDir();
+
+        LOGGER.info("Flushing writers in preparation for sphinx-build.");
+        context.writerDelegator().flushWriters();
+
+        // Python must be available to run sphinx
+        try {
+            LOGGER.info("Attempting to discover python3 in order to run sphinx.");
+            runCommand("python3 --version", baseDir);
+        } catch (CodegenException e) {
+            LOGGER.warning("Unable to find python3 on path. Skipping automatic HTML doc build.");
+            logManualBuildInstructions(context);
+            return;
+        }
+
+        // TODO: detect if the user's existing python environment can be used
+        // You can get a big JSON document describing the python environment from
+        // `pip inspect` that has all the information we need.
+        try {
+            // First, we create a virtualenv to install dependencies into. This is necessary
+            // to not pollute the user's environment.
+            runCommand("python3 -m venv venv", baseDir);
+
+            // Next, install the dependencies into the venv.
+            runCommand("./venv/bin/pip install -r requirements.txt", baseDir);
+
+            // Finally, run sphinx itself.
+            runCommand("./venv/bin/sphinx-build -M dirhtml content build", baseDir);
+
+            System.out.printf(normalizeNewlines("""
+                Successfully built HTML docs. They can be found in "%1$s".
+
+                Other output formats can also be built. A python virtual environment \
+                has been created at "%2$s" containing the build tools needed for \
+                manually building the docs in other formats. See the virtual \
+                environment docs for information on how to activate it: \
+                https://docs.python.org/3/library/venv.html#how-venvs-work
+
+                Once the environment is activated, run `make dirhtml` from "%3$s" to \
+                to build the docs, substituting dirhtml for whatever format you wish \
+                to build.
+
+                To build the docs without activating the virtual environment, simply \
+                run `./venv/bin/sphinx-build -M dirhtml content build` from "%3$s", \
+                similarly substituting dirhtml for your desired format.
+
+                See sphinx docs for other output formats you can choose: \
+                https://www.sphinx-doc.org/en/master/usage/builders/index.html
+
+                """),
+                baseDir.resolve("build/dirhtml"),
+                baseDir.resolve("venv"),
+                baseDir
+            );
+        } catch (CodegenException e) {
+            LOGGER.warning("Unable to automatically build HTML docs: " + e);
+            logManualBuildInstructions(context);
+        }
+    }
+
+    private void logManualBuildInstructions(DocGenerationContext context) {
+        // TODO: try to get this printed out in the projection section
+        System.out.printf(normalizeNewlines("""
+            To build the HTML docs manually, you need to first install the python \
+            dependencies. These can be found in the `requirements.txt` file in \
+            "%1$s". The easiest way to install these is by running `pip install \
+            -r requirements.txt`. Depending on your environment, you may need to \
+            instead install them from your system package manager, or another \
+            source.
+
+            Once the dependencies are installed, run `make dirhtml` from \
+            "%1$s". Other output formats can also be built. See sphinx docs for \
+            other output formats: \
+            https://www.sphinx-doc.org/en/master/usage/builders/index.html
+
+            """),
+            context.fileManifest().getBaseDir()
+        );
+    }
 }

smithy-docgen-core/src/main/java/software/amazon/smithy/docgen/core/sections/sphinx/RequirementsSection.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.docgen.core.sections.sphinx;
+
+import java.util.List;
+import software.amazon.smithy.docgen.core.DocGenerationContext;
+import software.amazon.smithy.utils.CodeSection;
+
+/**
+ * Generates a requirements file needed to install and run sphinx.
+ *
+ * <p>Any requirements added here will be installed in the environment used to
+ * automatically build the docs with {@code sphinx-build}.
+ *
+ * @param context The context used to generate documentation.
+ * @param requirements The requirements as a list of <a href="https://peps.python.org/pep-0508/">PEP 508</a> strings.
+ */
+public record RequirementsSection(DocGenerationContext context, List<String> requirements) implements CodeSection {
+}