[clang-doc][NFC] Lift Mustache template generation from HTML

clang-tools-extra/clang-doc/Generators.cpp

@@ -7,9 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "Generators.h"
+#include "support/File.h"
+#include "llvm/Support/TimeProfiler.h"
 
 LLVM_INSTANTIATE_REGISTRY(clang::doc::GeneratorRegistry)
 
+using namespace llvm;
+using namespace llvm::json;
+using namespace llvm::mustache;
+
 namespace clang {
 namespace doc {
 
@@ -42,6 +48,136 @@ std::string getTagType(TagTypeKind AS) {
   llvm_unreachable("Unknown TagTypeKind");
 }
 
+Error createFileOpenError(StringRef FileName, std::error_code EC) {
+  return createFileError("cannot open file " + FileName, EC);
+}
+
+Error MustacheGenerator::setupTemplate(
+    std::unique_ptr<MustacheTemplateFile> &Template, StringRef TemplatePath,
+    std::vector<std::pair<StringRef, StringRef>> Partials) {
+  auto T = MustacheTemplateFile::createMustacheFile(TemplatePath);
+  if (Error Err = T.takeError())
+    return Err;
+  Template = std::move(T.get());
+  for (const auto &[Name, FileName] : Partials)
+    if (auto Err = Template->registerPartialFile(Name, FileName))
+      return Err;
+  return Error::success();
+}
+
+Error MustacheGenerator::generateDocumentation(
+    StringRef RootDir, StringMap<std::unique_ptr<doc::Info>> Infos,
+    const clang::doc::ClangDocContext &CDCtx, std::string DirName) {
+  {
+    llvm::TimeTraceScope TS("Setup Templates");
+    if (auto Err = setupTemplateFiles(CDCtx))
+      return Err;
+  }
+
+  {
+    llvm::TimeTraceScope TS("Generate JSON for Mustache");
+    if (auto JSONGenerator = findGeneratorByName("json")) {
+      if (Error Err = JSONGenerator.get()->generateDocumentation(
+              RootDir, std::move(Infos), CDCtx))
+        return Err;
+    } else
+      return JSONGenerator.takeError();
+  }
+
+  SmallString<128> JSONPath;
+  sys::path::native(RootDir.str() + "/json", JSONPath);
+
+  {
+    llvm::TimeTraceScope TS("Iterate JSON files");
+    std::error_code EC;
+    sys::fs::recursive_directory_iterator JSONIter(JSONPath, EC);
+    std::vector<json::Value> JSONFiles;
+    JSONFiles.reserve(Infos.size());
+    if (EC)
+      return createStringError("Failed to create directory iterator.");
+
+    SmallString<128> DocsDirPath(RootDir.str() + '/' + DirName);
+    sys::path::native(DocsDirPath);
+    if (auto EC = sys::fs::create_directories(DocsDirPath))
+      return createFileError(DocsDirPath, EC);
+    while (JSONIter != sys::fs::recursive_directory_iterator()) {
+      // create the same directory structure in the docs format dir
+      if (JSONIter->type() == sys::fs::file_type::directory_file) {
+        SmallString<128> DocsClonedPath(JSONIter->path());
+        sys::path::replace_path_prefix(DocsClonedPath, JSONPath, DocsDirPath);
+        if (auto EC = sys::fs::create_directories(DocsClonedPath)) {
+          return createFileError(DocsClonedPath, EC);
+        }
+      }
+
+      if (EC)
+        return createFileError("Failed to iterate: " + JSONIter->path(), EC);
+
+      auto Path = StringRef(JSONIter->path());
+      if (!Path.ends_with(".json")) {
+        JSONIter.increment(EC);
+        continue;
+      }
+
+      auto File = MemoryBuffer::getFile(Path);
+      if (EC = File.getError(); EC) {
+        // TODO: Buffer errors to report later, look into using Clang
+        // diagnostics.
+        llvm::errs() << "Failed to open file: " << Path << " " << EC.message()
+                     << '\n';
+      }
+
+      auto Parsed = json::parse((*File)->getBuffer());
+      if (!Parsed)
+        return Parsed.takeError();
+      auto ValidJSON = Parsed.get();
+
+      std::error_code FileErr;
+      SmallString<128> DocsFilePath(JSONIter->path());
+      sys::path::replace_path_prefix(DocsFilePath, JSONPath, DocsDirPath);
+      sys::path::replace_extension(DocsFilePath, DirName);
+      raw_fd_ostream InfoOS(DocsFilePath, FileErr, sys::fs::OF_None);
+      if (FileErr)
+        return createFileOpenError(Path, FileErr);
+
+      auto RelativeRootPath = getRelativePathToRoot(DocsFilePath, DocsDirPath);
+      auto InfoTypeStr =
+          getInfoTypeStr(Parsed->getAsObject(), sys::path::stem(DocsFilePath));
+      if (!InfoTypeStr)
+        return InfoTypeStr.takeError();
+      if (Error Err = generateDocForJSON(*Parsed, InfoOS, CDCtx,
+                                         InfoTypeStr.get(), RelativeRootPath))
+        return Err;
+      JSONIter.increment(EC);
+    }
+  }
+
+  return Error::success();
+}
+
+Expected<std::string> MustacheGenerator::getInfoTypeStr(Object *Info,
+                                                        StringRef Filename) {
+  auto StrValue = (*Info)["InfoType"];
+  if (StrValue.kind() != json::Value::Kind::String)
+    return createStringError("JSON file '%s' does not contain key: 'InfoType'.",
+                             Filename.str().c_str());
+  auto ObjTypeStr = StrValue.getAsString();
+  if (!ObjTypeStr.has_value())
+    return createStringError(
+        "JSON file '%s' does not contain 'InfoType' field as a string.",
+        Filename.str().c_str());
+  return ObjTypeStr.value().str();
+}
+
+SmallString<128>
+MustacheGenerator::getRelativePathToRoot(StringRef PathToFile,
+                                         StringRef DocsRootPath) {
+  SmallString<128> PathVec(PathToFile);
+  // Remove filename, or else the relative path will have an extra "../"
+  sys::path::remove_filename(PathVec);
+  return computeRelativePath(DocsRootPath, PathVec);
+}
+
 llvm::Error Generator::createResources(ClangDocContext &CDCtx) {
   return llvm::Error::success();
 }

clang-tools-extra/clang-doc/Generators.h

@@ -14,6 +14,8 @@
 
 #include "Representation.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Mustache.h"
 #include "llvm/Support/Registry.h"
 
 namespace clang {
@@ -27,10 +29,9 @@ class Generator {
 
   // Write out the decl info for the objects in the given map in the specified
   // format.
-  virtual llvm::Error
-  generateDocs(StringRef RootDir,
-               llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
-               const ClangDocContext &CDCtx) = 0;
+  virtual llvm::Error generateDocumentation(
+      StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+      const ClangDocContext &CDCtx, std::string DirName = "") = 0;
 
   // This function writes a file with the index previously constructed.
   // It can be overwritten by any of the inherited generators.
@@ -52,6 +53,85 @@ findGeneratorByName(llvm::StringRef Format);
 
 std::string getTagType(TagTypeKind AS);
 
+llvm::Error createFileOpenError(StringRef FileName, std::error_code EC);
+
+class MustacheTemplateFile {
+  llvm::BumpPtrAllocator Allocator;
+  llvm::StringSaver Saver;
+  llvm::mustache::MustacheContext Ctx;
+  llvm::mustache::Template T;
+  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+
+public:
+  static Expected<std::unique_ptr<MustacheTemplateFile>>
+  createMustacheFile(StringRef FileName) {
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrError =
+        llvm::MemoryBuffer::getFile(FileName);
+    if (auto EC = BufferOrError.getError())
+      return createFileOpenError(FileName, EC);
+    return std::make_unique<MustacheTemplateFile>(
+        std::move(BufferOrError.get()));
+  }
+
+  llvm::Error registerPartialFile(StringRef Name, StringRef FileName) {
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrError =
+        llvm::MemoryBuffer::getFile(FileName);
+    if (auto EC = BufferOrError.getError())
+      return createFileOpenError(FileName, EC);
+
+    std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrError.get());
+    StringRef FileContent = Buffer->getBuffer();
+    T.registerPartial(Name.str(), FileContent.str());
+    return llvm::Error::success();
+  }
+
+  void render(llvm::json::Value &V, raw_ostream &OS) { T.render(V, OS); }
+
+  MustacheTemplateFile(std::unique_ptr<llvm::MemoryBuffer> &&B)
+      : Saver(Allocator), Ctx(Allocator, Saver), T(B->getBuffer(), Ctx),
+        Buffer(std::move(B)) {}
+};
+
+struct MustacheGenerator : public Generator {
+  Expected<std::string> getInfoTypeStr(llvm::json::Object *Info,
+                                       StringRef Filename);
+
+  /// Used to find the relative path from the file to the format's docs root.
+  /// Mainly used for the HTML resource paths.
+  SmallString<128> getRelativePathToRoot(StringRef PathToFile,
+                                         StringRef DocsRootPath);
+  virtual ~MustacheGenerator() = default;
+
+  /// Initializes the template files from disk and calls setupTemplate to
+  /// register partials
+  virtual llvm::Error setupTemplateFiles(const ClangDocContext &CDCtx) = 0;
+
+  /// Populates templates with data from JSON and calls any specifics for the
+  /// format. For example, for HTML it will render the paths for CSS and JS.
+  virtual llvm::Error generateDocForJSON(llvm::json::Value &JSON,
+                                         llvm::raw_fd_ostream &OS,
+                                         const ClangDocContext &CDCtx,
+                                         StringRef ObjectTypeStr,
+                                         StringRef RelativeRootPath) = 0;
+
+  /// Registers partials to templates.
+  llvm::Error
+  setupTemplate(std::unique_ptr<MustacheTemplateFile> &Template,
+                StringRef TemplatePath,
+                std::vector<std::pair<StringRef, StringRef>> Partials);
+
+  /// \brief The main orchestrator for Mustache-based documentation.
+  ///
+  /// 1. Initializes templates files from disk by calling setupTemplateFiles.
+  /// 2. Calls the JSON generator to write JSON to disk.
+  /// 3. Iterates over the JSON files, recreates the directory structure from
+  /// JSON, and calls generateDocForJSON for each file.
+  /// 4. A file of the desired format is created.
+  llvm::Error generateDocumentation(
+      StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+      const clang::doc::ClangDocContext &CDCtx, std::string DirName) override;
+};
+
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the generators.
 extern volatile int YAMLGeneratorAnchorSource;

clang-tools-extra/clang-doc/HTMLGenerator.cpp

@@ -897,20 +897,19 @@ class HTMLGenerator : public Generator {
 public:
   static const char *Format;
 
-  llvm::Error generateDocs(StringRef RootDir,
-                           llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
-                           const ClangDocContext &CDCtx) override;
+  llvm::Error generateDocumentation(
+      StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+      const ClangDocContext &CDCtx, std::string DirName) override;
   llvm::Error createResources(ClangDocContext &CDCtx) override;
   llvm::Error generateDocForInfo(Info *I, llvm::raw_ostream &OS,
                                  const ClangDocContext &CDCtx) override;
 };
 
 const char *HTMLGenerator::Format = "html";
 
-llvm::Error
-HTMLGenerator::generateDocs(StringRef RootDir,
-                            llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
-                            const ClangDocContext &CDCtx) {
+llvm::Error HTMLGenerator::generateDocumentation(
+    StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+    const ClangDocContext &CDCtx, std::string DirName) {
   // Track which directories we already tried to create.
   llvm::StringSet<> CreatedDirs;