For HTML files with known tokens, generate a TOC. (#1)

Author: chinmaygarde

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "third_party/tinyxml2"]
 	path = third_party/tinyxml2
 	url = https://github.com/leethomason/tinyxml2.git
+[submodule "third_party/gumbo/gumbo"]
+	path = third_party/gumbo/gumbo
+	url = https://github.com/google/gumbo-parser.git

CMakeLists.txt

@@ -12,6 +12,7 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build deps as a static library." FORCE)
 add_subdirectory("third_party/googletest")
 add_subdirectory("third_party/tinyxml2")
 add_subdirectory("third_party/sqlite3")
+add_subdirectory("third_party/gumbo")
 
 add_subdirectory("source")
 add_subdirectory("tests")

source/CMakeLists.txt

@@ -14,12 +14,15 @@ add_library(doxygen2docset_lib
     "token.h"
     "token_parser.cc"
     "token_parser.h"
+    "html_parser.h"
+    "html_parser.cc"
 )
 
 target_link_libraries(doxygen2docset_lib
   PUBLIC
     tinyxml2
     sqlite3
+    gumbo
 )
 
 target_include_directories(doxygen2docset_lib

source/builder.cc

@@ -4,6 +4,7 @@
 
 #include "docset_index.h"
 #include "file.h"
+#include "html_parser.h"
 #include "logger.h"
 #include "plist_parser.h"
 #include "token_parser.h"
@@ -46,7 +47,9 @@ bool BuildDocset(const std::string& docs, const std::string& location) {
     return false;
   }
 
-  if (!index.AddTokens(token_parser.ReadTokens())) {
+  auto tokens = token_parser.ReadTokens();
+
+  if (!index.AddTokens(tokens)) {
     D2D_ERROR << "Could not add tokens to docset index.";
     return false;
   }
@@ -60,8 +63,44 @@ bool BuildDocset(const std::string& docs, const std::string& location) {
       "Tokens.xml",
       "Makefile",
   };
-  auto predicate = [&filtered](const std::string& file_name) -> bool {
-    return filtered.count(file_name) == 0;
+
+  auto tokens_by_file = Token::GetTokensByFile(tokens);
+
+  auto predicate = [&filtered, &tokens_by_file](
+                       const std::string& from_file_name,  //
+                       const struct stat& from_stat,       //
+                       const AutoFD& from_fd,              //
+                       const std::string& to_file_name) -> bool {
+    // Check if this file needs to be filtered away.
+    if (filtered.count(from_file_name) != 0) {
+      return true;
+    }
+
+    // Check if this is a file in which a TOC needs to be generated.
+    {
+      const auto found = tokens_by_file.find(from_file_name);
+      if (found != tokens_by_file.end()) {
+        HTMLParser parser(OpenFileReadOnly(from_fd, from_stat.st_size));
+        auto html_with_toc = parser.BuildHTMLWithTOC(found->second);
+        if (html_with_toc.IsValid()) {
+          if (!CopyData(html_with_toc.Get(),      //
+                        html_with_toc.GetSize(),  //
+                        to_file_name)) {
+            D2D_ERROR << "Could not copy HTML with TOC to " << to_file_name
+                      << ". Will try moving file without TOC.";
+
+          } else {
+            return true;
+          }
+        } else {
+          D2D_ERROR << "Could not build TOC in file: " << from_file_name
+                    << ". Skipping.";
+        }
+      }
+    }
+
+    // Copy file as-is.
+    return CopyFile(from_stat, from_fd, to_file_name);
   };
 
   if (!CopyFiles(docs, documents_directory, predicate)) {

source/file.cc

@@ -1,97 +1,12 @@
 #include "file.h"
 
-#include <dirent.h>
-#include <fcntl.h>
 #include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
 
 #include <algorithm>
 #include <sstream>
 
-#include "logger.h"
-
 namespace d2d {
 
-class AutoFD {
- public:
-  AutoFD(int fd) : fd_(fd) {}
-
-  AutoFD(AutoFD&& fd) = delete;
-
-  AutoFD(const AutoFD& fd) = delete;
-
-  AutoFD& operator=(const AutoFD&) = delete;
-
-  AutoFD& operator=(AutoFD&& other) {
-    Reset(other.fd_);
-    other.fd_ = -1;
-    return *this;
-  };
-
-  int Get() const { return fd_; };
-
-  bool IsValid() const { return fd_ > 0; };
-
-  void Reset(int fd = -1) {
-    if (fd_ != fd && fd_ > 0) {
-      int result = D2D_TEMP_FAILURE_RETRY(::close(fd_));
-      if (result == -1) {
-        D2D_ERROR << "Could not close a file descriptor.";
-      }
-    }
-    fd_ = fd;
-  }
-
-  ~AutoFD() { Reset(); }
-
- private:
-  int fd_ = -1;
-};
-
-class AutoDir {
- public:
-  AutoDir(DIR* dir) : dir_(dir) {}
-
-  DIR* Get() const { return dir_; }
-
-  bool IsValid() const { return dir_ != nullptr; }
-
-  ~AutoDir() {
-    if (dir_ != nullptr) {
-      ::closedir(dir_);
-    }
-  }
-
- private:
-  DIR* dir_ = nullptr;
-  D2D_DISALLOW_COPY_AND_ASSIGN(AutoDir);
-};
-
-class AutoMapping {
- public:
-  AutoMapping(void* mapping, size_t size) : mapping_(mapping), size_(size) {}
-
-  ~AutoMapping() {
-    if (mapping_ != MAP_FAILED) {
-      if (::munmap(mapping_, size_) != 0) {
-        D2D_ERROR << "Error unmapping file.";
-      }
-    }
-  }
-
-  void* Get() const { return mapping_; }
-
-  bool IsValid() const { return mapping_ != MAP_FAILED; }
-
- private:
-  void* mapping_ = MAP_FAILED;
-  size_t size_ = 0;
-
-  D2D_DISALLOW_COPY_AND_ASSIGN(AutoMapping);
-};
-
 bool MakeDirectories(const std::vector<std::string>& directories) {
   AutoFD current_level(AT_FDCWD);
 
@@ -121,7 +36,8 @@ bool MakeDirectories(const std::vector<std::string>& directories) {
   return true;
 }
 
-bool CopyData(const void* data, size_t length, const std::string& to_path) {
+bool CopyData(const void* from_data, size_t from_length,
+              const std::string& to_path) {
   AutoFD to_file(
       D2D_TEMP_FAILURE_RETRY(::open(to_path.c_str(), O_CREAT | O_TRUNC | O_RDWR,
                                     S_IRUSR | S_IWUSR | S_IXUSR)));
@@ -131,31 +47,31 @@ bool CopyData(const void* data, size_t length, const std::string& to_path) {
     return false;
   }
 
-  if (::ftruncate(to_file.Get(), length) != 0) {
+  if (::ftruncate(to_file.Get(), from_length) != 0) {
     D2D_ERROR << "Could not truncate file " << to_path;
     return false;
   }
-  AutoMapping to_mapping(::mmap(nullptr, length, PROT_WRITE,
+  AutoMapping to_mapping(::mmap(nullptr, from_length, PROT_WRITE,
                                 MAP_FILE | MAP_SHARED, to_file.Get(), 0),
-                         length);
+                         from_length);
 
   if (!to_mapping.IsValid()) {
     D2D_ERROR << "Could not setup mapping to perform file copy.";
     return false;
   }
 
-  ::memcpy(to_mapping.Get(), data, length);
+  ::memcpy(to_mapping.Get(), from_data, from_length);
 
-  if (::msync(to_mapping.Get(), length, MS_SYNC) != 0) {
+  if (::msync(to_mapping.Get(), from_length, MS_SYNC) != 0) {
     D2D_ERROR << "Could not sync file contents.";
     return false;
   }
 
   return true;
 }
 
-static bool CopyFile(const struct stat& from_stat, const AutoFD& from,
-                     const std::string& to_path) {
+bool CopyFile(const struct stat& from_stat, const AutoFD& from,
+              const std::string& to_path) {
   AutoMapping from_mapping(::mmap(nullptr, from_stat.st_size, PROT_READ,
                                   MAP_FILE | MAP_PRIVATE, from.Get(), 0),
                            from_stat.st_size);
@@ -212,10 +128,6 @@ bool CopyFiles(const std::string& from_path,
       continue;
     }
 
-    if (!predicate(file_name)) {
-      continue;
-    }
-
     AutoFD from_fd(D2D_TEMP_FAILURE_RETRY(
         ::openat(::dirfd(from.Get()), file_name.c_str(), O_RDONLY)));
     if (!from_fd.IsValid()) {
@@ -238,7 +150,11 @@ bool CopyFiles(const std::string& from_path,
         return false;
       }
     } else {
-      if (!CopyFile(from_stat, from_fd, JoinPaths(to_path, file_name))) {
+      if (!predicate(file_name,                     //
+                     from_stat,                     //
+                     from_fd,                       //
+                     JoinPaths(to_path, file_name)  //
+                     )) {
         D2D_ERROR << "Could not copy file " << file_name;
         return false;
       }
@@ -284,4 +200,48 @@ std::string JoinPaths(const std::vector<std::string>& paths,
   return JoinPaths(merged);
 }
 
+std::unique_ptr<AutoMapping> OpenFileReadOnly(const AutoFD& fd, size_t size) {
+  if (!fd.IsValid()) {
+    D2D_ERROR << "File descriptor was invalid";
+    return nullptr;
+  }
+
+  auto mapping =
+      std::make_unique<AutoMapping>(::mmap(nullptr,                 //
+                                           size,                    //
+                                           PROT_READ,               //
+                                           MAP_FILE | MAP_PRIVATE,  //
+                                           fd.Get(),                //
+                                           0),
+                                    size);
+  if (!mapping || !mapping->IsValid()) {
+    D2D_ERROR << "Could not create file mapping.";
+    return nullptr;
+  }
+
+  return mapping;
+}
+
+std::unique_ptr<AutoMapping> OpenFileReadOnly(const std::string& path) {
+  if (path.size() == 0) {
+    D2D_ERROR << "Path was empty when attempting to open file.";
+    return nullptr;
+  }
+
+  AutoFD fd(D2D_TEMP_FAILURE_RETRY(::open(path.c_str(), O_RDONLY)));
+
+  if (!fd.IsValid()) {
+    D2D_ERROR << "Could not open file: " << path;
+    return nullptr;
+  }
+
+  struct stat stat_buf = {0};
+  if (::fstat(fd.Get(), &stat_buf) != 0) {
+    D2D_ERROR << "Could not stat file.";
+    return nullptr;
+  }
+
+  return OpenFileReadOnly(fd, stat_buf.st_size);
+}
+
 }  // namespace d2d