[clang-doc] revert to a nested directory structure (llvm#166978)

Restore previous behavior of creating a nested directory structure in
JSON and HTML output. Now, namespaces are all named index.json and will
serve as the main page of their directory, e.g. clang/index.json is the
"homepage" for the namespace clang.

Some new functionality is added to replicate the directory structure
from JSON to the HTML output, since the HTML output doesn't handle any
Infos to create the directory structure.

Author: evelez7

clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp

@@ -29,7 +29,8 @@ namespace clang {
 namespace doc {
 static Error generateDocForJSON(json::Value &JSON, StringRef Filename,
                                 StringRef Path, raw_fd_ostream &OS,
-                                const ClangDocContext &CDCtx);
+                                const ClangDocContext &CDCtx,
+                                StringRef HTMLRootPath);
 
 static Error createFileOpenError(StringRef FileName, std::error_code EC) {
   return createFileError("cannot open file " + FileName, EC);
@@ -159,16 +160,24 @@ Error MustacheHTMLGenerator::generateDocs(
   {
     llvm::TimeTraceScope TS("Iterate JSON files");
     std::error_code EC;
-    sys::fs::directory_iterator JSONIter(JSONPath, EC);
+    sys::fs::recursive_directory_iterator JSONIter(JSONPath, EC);
     std::vector<json::Value> JSONFiles;
     JSONFiles.reserve(Infos.size());
     if (EC)
       return createStringError("Failed to create directory iterator.");
 
-    SmallString<128> HTMLDirPath(RootDir.str() + "/html/");
+    SmallString<128> HTMLDirPath(RootDir.str() + "/html");
     if (auto EC = sys::fs::create_directories(HTMLDirPath))
       return createFileError(HTMLDirPath, EC);
-    while (JSONIter != sys::fs::directory_iterator()) {
+    while (JSONIter != sys::fs::recursive_directory_iterator()) {
+      // create the same directory structure in the HTML dir
+      if (JSONIter->type() == sys::fs::file_type::directory_file) {
+        SmallString<128> HTMLClonedPath(JSONIter->path());
+        sys::path::replace_path_prefix(HTMLClonedPath, JSONPath, HTMLDirPath);
+        if (auto EC = sys::fs::create_directories(HTMLClonedPath))
+          return createFileError(HTMLClonedPath, EC);
+      }
+
       if (EC)
         return createFileError("Failed to iterate: " + JSONIter->path(), EC);
 
@@ -190,15 +199,16 @@ Error MustacheHTMLGenerator::generateDocs(
         return Parsed.takeError();
 
       std::error_code FileErr;
-      SmallString<128> HTMLFilePath(HTMLDirPath);
-      sys::path::append(HTMLFilePath, sys::path::filename(Path));
+      SmallString<128> HTMLFilePath(JSONIter->path());
+      sys::path::replace_path_prefix(HTMLFilePath, JSONPath, HTMLDirPath);
       sys::path::replace_extension(HTMLFilePath, "html");
       raw_fd_ostream InfoOS(HTMLFilePath, FileErr, sys::fs::OF_None);
       if (FileErr)
         return createFileOpenError(Path, FileErr);
 
-      if (Error Err = generateDocForJSON(*Parsed, sys::path::stem(HTMLFilePath),
-                                         HTMLFilePath, InfoOS, CDCtx))
+      if (Error Err =
+              generateDocForJSON(*Parsed, sys::path::stem(HTMLFilePath),
+                                 HTMLFilePath, InfoOS, CDCtx, HTMLDirPath))
         return Err;
       JSONIter.increment(EC);
     }
@@ -207,16 +217,16 @@ Error MustacheHTMLGenerator::generateDocs(
   return Error::success();
 }
 
-static Error setupTemplateValue(const ClangDocContext &CDCtx, json::Value &V) {
+static Error setupTemplateValue(const ClangDocContext &CDCtx, json::Value &V,
+                                SmallString<128> RelativeHTMLPath) {
   V.getAsObject()->insert({"ProjectName", CDCtx.ProjectName});
   json::Value StylesheetArr = Array();
-  SmallString<128> RelativePath("./");
-  sys::path::native(RelativePath, sys::path::Style::posix);
+  sys::path::native(RelativeHTMLPath, sys::path::Style::posix);
 
   auto *SSA = StylesheetArr.getAsArray();
   SSA->reserve(CDCtx.UserStylesheets.size());
   for (const auto &FilePath : CDCtx.UserStylesheets) {
-    SmallString<128> StylesheetPath = RelativePath;
+    SmallString<128> StylesheetPath = RelativeHTMLPath;
     sys::path::append(StylesheetPath, sys::path::Style::posix,
                       sys::path::filename(FilePath));
     SSA->emplace_back(StylesheetPath);
@@ -227,7 +237,7 @@ static Error setupTemplateValue(const ClangDocContext &CDCtx, json::Value &V) {
   auto *SCA = ScriptArr.getAsArray();
   SCA->reserve(CDCtx.JsScripts.size());
   for (auto Script : CDCtx.JsScripts) {
-    SmallString<128> JsPath = RelativePath;
+    SmallString<128> JsPath = RelativeHTMLPath;
     sys::path::append(JsPath, sys::path::Style::posix,
                       sys::path::filename(Script));
     SCA->emplace_back(JsPath);
@@ -238,7 +248,8 @@ static Error setupTemplateValue(const ClangDocContext &CDCtx, json::Value &V) {
 
 static Error generateDocForJSON(json::Value &JSON, StringRef Filename,
                                 StringRef Path, raw_fd_ostream &OS,
-                                const ClangDocContext &CDCtx) {
+                                const ClangDocContext &CDCtx,
+                                StringRef HTMLRootPath) {
   auto StrValue = (*JSON.getAsObject())["InfoType"];
   if (StrValue.kind() != json::Value::Kind::String)
     return createStringError("JSON file '%s' does not contain key: 'InfoType'.",
@@ -249,13 +260,17 @@ static Error generateDocForJSON(json::Value &JSON, StringRef Filename,
         "JSON file '%s' does not contain 'InfoType' field as a string.",
         Filename.str().c_str());
 
+  SmallString<128> PathVec(Path);
+  // Remove filename, or else the relative path will have an extra "../"
+  sys::path::remove_filename(PathVec);
+  auto RelativeHTMLPath = computeRelativePath(HTMLRootPath, PathVec);
   if (ObjTypeStr.value() == "namespace") {
-    if (auto Err = setupTemplateValue(CDCtx, JSON))
+    if (auto Err = setupTemplateValue(CDCtx, JSON, RelativeHTMLPath))
       return Err;
     assert(NamespaceTemplate && "NamespaceTemplate is nullptr.");
     NamespaceTemplate->render(JSON, OS);
   } else if (ObjTypeStr.value() == "record") {
-    if (auto Err = setupTemplateValue(CDCtx, JSON))
+    if (auto Err = setupTemplateValue(CDCtx, JSON, RelativeHTMLPath))
       return Err;
     assert(RecordTemplate && "RecordTemplate is nullptr.");
     RecordTemplate->render(JSON, OS);

clang-tools-extra/clang-doc/JSONGenerator.cpp

@@ -581,22 +581,14 @@ static SmallString<16> determineFileName(Info *I, SmallString<128> &Path) {
   if (I->IT == InfoType::IT_record) {
     auto *RecordSymbolInfo = static_cast<SymbolInfo *>(I);
     FileName = RecordSymbolInfo->MangledName;
-  } else if (I->USR == GlobalNamespaceID)
+  } else if (I->IT == InfoType::IT_namespace) {
     FileName = "index";
-  else if (I->IT == InfoType::IT_namespace) {
-    for (const auto &NS : I->Namespace) {
-      FileName += NS.Name;
-      FileName += "_";
-    }
-    FileName += I->Name;
   } else
     FileName = I->Name;
   sys::path::append(Path, FileName + ".json");
   return FileName;
 }
 
-// FIXME: Revert back to creating nested directories for namespaces instead of
-// putting everything in a flat directory structure.
 Error JSONGenerator::generateDocs(
     StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
     const ClangDocContext &CDCtx) {
@@ -609,6 +601,7 @@ Error JSONGenerator::generateDocs(
     auto RootDirStr = RootDir.str() + "/json";
     StringRef JSONDir = StringRef(RootDirStr);
     sys::path::native(JSONDir, Path);
+    sys::path::append(Path, Info->getRelativeFilePath(""));
     if (!CreatedDirs.contains(Path)) {
       if (std::error_code Err = sys::fs::create_directories(Path);
           Err != std::error_code())

clang-tools-extra/test/clang-doc/basic-project.mustache.test

@@ -2,17 +2,17 @@
 // RUN: sed 's|$test_dir|%/S|g' %S/Inputs/basic-project/database_template.json > %t/build/compile_commands.json
 
 // RUN: clang-doc --format=mustache --output=%t/docs --executor=all-TUs %t/build/compile_commands.json
-// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV5Shape.html -check-prefix=HTML-SHAPE
-// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV10Calculator.html -check-prefix=HTML-CALC
-// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV9Rectangle.html -check-prefix=HTML-RECTANGLE
-// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV6Circle.html -check-prefix=HTML-CIRCLE
+// RUN: FileCheck %s -input-file=%t/docs/html/GlobalNamespace/_ZTV5Shape.html -check-prefix=HTML-SHAPE
+// RUN: FileCheck %s -input-file=%t/docs/html/GlobalNamespace/_ZTV10Calculator.html -check-prefix=HTML-CALC
+// RUN: FileCheck %s -input-file=%t/docs/html/GlobalNamespace/_ZTV9Rectangle.html -check-prefix=HTML-RECTANGLE
+// RUN: FileCheck %s -input-file=%t/docs/html/GlobalNamespace/_ZTV6Circle.html -check-prefix=HTML-CIRCLE
 
 HTML-SHAPE: <html lang="en-US">
 HTML-SHAPE: <head>
 HTML-SHAPE:     <meta charset="utf-8"/>
 HTML-SHAPE:     <title>Shape</title>
-HTML-SHAPE:         <link rel="stylesheet" type="text/css" href="./clang-doc-mustache.css"/>
-HTML-SHAPE:         <script src="./mustache-index.js"></script>
+HTML-SHAPE:         <link rel="stylesheet" type="text/css" href="../clang-doc-mustache.css"/>
+HTML-SHAPE:         <script src="../mustache-index.js"></script>
 HTML-SHAPE:     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">
 HTML-SHAPE:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
 HTML-SHAPE:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/cpp.min.js"></script>
@@ -151,8 +151,8 @@ HTML-CALC: <html lang="en-US">
 HTML-CALC: <head>
 HTML-CALC:     <meta charset="utf-8"/>
 HTML-CALC:     <title>Calculator</title>
-HTML-CALC:         <link rel="stylesheet" type="text/css" href="./clang-doc-mustache.css"/>
-HTML-CALC:         <script src="./mustache-index.js"></script>
+HTML-CALC:         <link rel="stylesheet" type="text/css" href="../clang-doc-mustache.css"/>
+HTML-CALC:         <script src="../mustache-index.js"></script>
 HTML-CALC:     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">
 HTML-CALC:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
 HTML-CALC:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/cpp.min.js"></script>
@@ -440,8 +440,8 @@ HTML-RECTANGLE: <html lang="en-US">
 HTML-RECTANGLE: <head>
 HTML-RECTANGLE:     <meta charset="utf-8"/>
 HTML-RECTANGLE:     <title>Rectangle</title>
-HTML-RECTANGLE:         <link rel="stylesheet" type="text/css" href="./clang-doc-mustache.css"/>
-HTML-RECTANGLE:         <script src="./mustache-index.js"></script>
+HTML-RECTANGLE:         <link rel="stylesheet" type="text/css" href="../clang-doc-mustache.css"/>
+HTML-RECTANGLE:         <script src="../mustache-index.js"></script>
 HTML-RECTANGLE:     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">
 HTML-RECTANGLE:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
 HTML-RECTANGLE:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/cpp.min.js"></script>
@@ -597,8 +597,8 @@ HTML-CIRCLE: <html lang="en-US">
 HTML-CIRCLE: <head>
 HTML-CIRCLE:     <meta charset="utf-8"/>
 HTML-CIRCLE:     <title>Circle</title>
-HTML-CIRCLE:         <link rel="stylesheet" type="text/css" href="./clang-doc-mustache.css"/>
-HTML-CIRCLE:         <script src="./mustache-index.js"></script>
+HTML-CIRCLE:         <link rel="stylesheet" type="text/css" href="../clang-doc-mustache.css"/>
+HTML-CIRCLE:         <script src="../mustache-index.js"></script>
 HTML-CIRCLE:     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">
 HTML-CIRCLE:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
 HTML-CIRCLE:     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/cpp.min.js"></script>

clang-tools-extra/test/clang-doc/json/class-requires.cpp

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/GlobalNamespace/_ZTV7MyClass.json
 
 template<typename T>
 concept Addable = requires(T a, T b) {

clang-tools-extra/test/clang-doc/json/class-specialization.cpp

@@ -1,7 +1,7 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json --check-prefix=BASE
-// RUN: FileCheck %s < %t/json/_ZTV7MyClassIiE.json --check-prefix=SPECIALIZATION
+// RUN: FileCheck %s < %t/json/GlobalNamespace/_ZTV7MyClass.json --check-prefix=BASE
+// RUN: FileCheck %s < %t/json/GlobalNamespace/_ZTV7MyClassIiE.json --check-prefix=SPECIALIZATION
 
 template<typename T> struct MyClass {};
 

clang-tools-extra/test/clang-doc/json/class-template.cpp

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/GlobalNamespace/_ZTV7MyClass.json
 
 template<typename T> struct MyClass {
   T MemberTemplate;

clang-tools-extra/test/clang-doc/json/class.cpp

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/GlobalNamespace/_ZTV7MyClass.json
 
 struct Foo;
 

clang-tools-extra/test/clang-doc/json/compound-constraints.cpp

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/index.json
+// RUN: FileCheck %s < %t/json/GlobalNamespace/index.json
 
 template<typename T> concept Incrementable = requires (T a) {
   a++;

clang-tools-extra/test/clang-doc/json/concept.cpp

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/index.json
+// RUN: FileCheck %s < %t/json/GlobalNamespace/index.json
 
 // Requires that T suports post and pre-incrementing.
 template<typename T>

clang-tools-extra/test/clang-doc/json/function-requires.cpp

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/json/index.json
+// RUN: FileCheck %s < %t/json/GlobalNamespace/index.json
 
 template<typename T>
 concept Incrementable = requires(T x) {