feat!: support recursive path build (#1260)

Author: ken-matsui

Dockerfile

@@ -19,6 +19,7 @@ COPY Makefile .
 COPY include ./include/
 COPY lib ./lib/
 COPY src ./src/
+COPY semver ./semver/
 RUN make BUILD=release install
 
 FROM $base AS runtime

Makefile

@@ -44,7 +44,7 @@ DEFINES := -DCABIN_CABIN_PKG_VERSION='"$(VERSION)"' \
   -DCABIN_CABIN_COMMIT_HASH='"$(COMMIT_HASH)"' \
   -DCABIN_CABIN_COMMIT_SHORT_HASH='"$(COMMIT_SHORT_HASH)"' \
   -DCABIN_CABIN_COMMIT_DATE='"$(COMMIT_DATE)"'
-INCLUDES := -Iinclude -Isrc -isystem $(O)/DEPS/toml11/include \
+INCLUDES := -Iinclude -Isrc -Isemver/include -isystem $(O)/DEPS/toml11/include \
   -isystem $(O)/DEPS/mitama-cpp-result/include \
   -isystem $(O)/DEPS/rs-cpp/include
 
@@ -63,7 +63,8 @@ endif
 LDLIBS := $(PKG_LIBS)
 
 # Source files
-SRCS := $(shell find src -name '*.cc') $(shell find lib -name '*.cc')
+SRCS := $(shell find src -name '*.cc') $(shell find lib -name '*.cc') \
+  $(shell find semver/lib -name '*.cc')
 OBJS := $(SRCS:%.cc=$(O)/%.o)
 DEPS := $(OBJS:.o=.d)
 

cabin.toml

@@ -19,6 +19,7 @@ libgit2 = {version = ">=1.7 && <1.10", system = true}
 nlohmann_json = {version = "3.10.5", system = true}
 tbb = {version = ">=2021.5.0 && <2023.0.0", system = true}
 rs-cpp = {git = "https://github.com/ken-matsui/rs-cpp.git", branch = "main"}
+semver = {path = "semver"}
 
 [dev-dependencies]
 boost-ut = {git = "https://github.com/boost-ext/ut.git", tag = "v2.3.1"}

include/Builder/BuildGraph.hpp

@@ -46,7 +46,7 @@ class BuildGraph {
   const std::string& libraryName() const { return libName; }
   const std::vector<TestTarget>& testTargets() const { return testTargets_; }
 
-  rs::Result<void> installDeps(bool includeDevDeps);
+  rs::Result<void> installDeps(bool includeDevDeps, bool suppressDepDiag);
   void enableCoverage();
   rs::Result<void> plan(bool logAnalysis = true);
   rs::Result<void> writeBuildFilesIfNeeded() const;
@@ -123,6 +123,7 @@ class BuildGraph {
   std::unordered_map<std::string, CompileUnit> compileUnits;
   std::vector<TestTarget> testTargets_;
   std::unordered_set<std::string> srcObjectTargets;
+  std::unordered_set<std::string> libSupportObjects;
   std::string archiver = "ar";
 
   std::string cxxFlags;

include/Manifest.hpp

@@ -129,7 +129,9 @@ class Manifest {
   static rs::Result<fs::path>
   findPath(fs::path candidateDir = fs::current_path()) noexcept;
 
-  rs::Result<std::vector<CompilerOpts>> installDeps(bool includeDevDeps) const;
+  rs::Result<std::vector<CompilerOpts>>
+  installDeps(bool includeDevDeps, const BuildProfile& buildProfile,
+              bool suppressDepDiag = false) const;
 
 private:
   Manifest(fs::path path, Package package, std::vector<Dependency> dependencies,

lib/Builder/BuildGraph.cc

@@ -128,10 +128,11 @@ bool BuildGraph::isUpToDate(const std::string_view fileName) const {
   }
 
   const fs::file_time_type configTime = fs::last_write_time(filePath);
-  const std::array<fs::path, 3> watchedDirs{
+  const std::array<fs::path, 4> watchedDirs{
     project.rootPath / "src",
     project.rootPath / "lib",
     project.rootPath / "include",
+    project.rootPath / "tests",
   };
   for (const fs::path& dir : watchedDirs) {
     if (!fs::exists(dir)) {
@@ -432,6 +433,10 @@ BuildGraph::processUnittestSrc(const fs::path& sourceFilePath,
 
   std::vector<std::string> linkInputs;
   linkInputs.push_back(testObjTarget);
+  std::unordered_set<std::string> linkSeen(linkInputs.begin(),
+                                           linkInputs.end());
+
+  const std::string underTest = sourceFilePath.stem().string();
 
   if (isSrcUnit) {
     std::unordered_set<std::string> deps;
@@ -441,6 +446,16 @@ BuildGraph::processUnittestSrc(const fs::path& sourceFilePath,
     std::vector<std::string> srcDeps(deps.begin(), deps.end());
     std::ranges::sort(srcDeps);
     linkInputs.insert(linkInputs.end(), srcDeps.begin(), srcDeps.end());
+    linkSeen.insert(srcDeps.begin(), srcDeps.end());
+  }
+
+  for (const std::string& obj : libSupportObjects) {
+    if (fs::path(obj).stem() == underTest) {
+      continue;
+    }
+    if (linkSeen.insert(obj).second) {
+      linkInputs.push_back(obj);
+    }
   }
 
   if (hasLibraryTarget_) {
@@ -486,10 +501,22 @@ BuildGraph::processIntegrationTestSrc(const fs::path& sourceFilePath,
       fs::relative(testBinaryPath, outBasePath_).generic_string();
 
   std::vector<std::string> linkInputs{ testObjTarget };
+  std::unordered_set<std::string> linkSeen(linkInputs.begin(),
+                                           linkInputs.end());
+
+  const std::string underTest = sourceFilePath.stem().string();
+  for (const std::string& obj : libSupportObjects) {
+    if (fs::path(obj).stem() == underTest) {
+      continue;
+    }
+    if (linkSeen.insert(obj).second) {
+      linkInputs.push_back(obj);
+    }
+  }
+
   if (hasLibraryTarget_) {
     linkInputs.push_back(libName);
   }
-  std::ranges::sort(linkInputs);
 
   NinjaEdge linkEdge;
   linkEdge.outputs = { testBinary };
@@ -547,9 +574,11 @@ void BuildGraph::collectBinDepObjs(
   }
 }
 
-rs::Result<void> BuildGraph::installDeps(const bool includeDevDeps) {
+rs::Result<void> BuildGraph::installDeps(const bool includeDevDeps,
+                                         const bool suppressDepDiag) {
   const std::vector<CompilerOpts> depsCompOpts =
-      rs_try(project.manifest.installDeps(includeDevDeps));
+      rs_try(project.manifest.installDeps(includeDevDeps, buildProfile_,
+                                          suppressDepDiag));
 
   for (const CompilerOpts& depOpts : depsCompOpts) {
     project.compilerOpts.merge(depOpts);
@@ -632,12 +661,6 @@ rs::Result<void> BuildGraph::configure() {
   }
   hasLibraryTarget_ = !publicSourceFilePaths.empty();
 
-  if (!hasBinaryTarget_ && !hasLibraryTarget_) {
-    rs_bail("expected either `src/main{}` or at least one source file under "
-            "`lib/` matching {}",
-            SOURCE_FILE_EXTS, SOURCE_FILE_EXTS);
-  }
-
   const SourceRoot srcRoot(srcDir);
   const SourceRoot libRoot(libDir, fs::path("lib"));
 
@@ -656,6 +679,19 @@ rs::Result<void> BuildGraph::configure() {
   std::unordered_set<std::string> buildObjTargets = srcObjTargets;
   buildObjTargets.insert(libObjTargets.begin(), libObjTargets.end());
 
+  libSupportObjects.clear();
+  for (const std::string& libObj : libObjTargets) {
+    const auto it = compileUnits.find(libObj);
+    if (it == compileUnits.end()) {
+      continue;
+    }
+    collectBinDepObjs(libSupportObjects, "", it->second.dependencies,
+                      buildObjTargets);
+  }
+  std::erase_if(libSupportObjects, [this](const std::string& obj) {
+    return !srcObjectTargets.contains(obj);
+  });
+
   if (hasBinaryTarget_) {
     const fs::path mainObjPath = project.buildOutPath / "main.o";
     const std::string mainObj =

lib/Manifest.cc

@@ -1,7 +1,9 @@
 #include "Manifest.hpp"
 
 #include "Builder/BuildProfile.hpp"
+#include "Builder/Builder.hpp"
 #include "Builder/Compiler.hpp"
+#include "Diag.hpp"
 #include "Semver.hpp"
 #include "TermColor.hpp"
 #include "VersionReq.hpp"
@@ -103,14 +105,16 @@ static fs::path canonicalizePathDep(const fs::path& baseDir,
 }
 
 static rs::Result<void>
-installDependencies(const Manifest& manifest, bool includeDevDeps,
+installDependencies(const Manifest& manifest, const BuildProfile& buildProfile,
+                    bool includeDevDeps, bool suppressDepDiag,
                     std::unordered_map<std::string, DepKey>& seenDeps,
                     std::unordered_set<fs::path>& visited,
                     std::vector<CompilerOpts>& installed);
 
 static rs::Result<void>
 installPathDependency(const Manifest& manifest, const PathDependency& pathDep,
-                      bool includeDevDeps,
+                      const BuildProfile& buildProfile, bool includeDevDeps,
+                      bool suppressDepDiag,
                       std::unordered_map<std::string, DepKey>& seenDeps,
                       std::unordered_set<fs::path>& visited,
                       std::vector<CompilerOpts>& installed) {
@@ -124,22 +128,58 @@ installPathDependency(const Manifest& manifest, const PathDependency& pathDep,
   }
 
   CompilerOpts pathOpts;
-  pathOpts.cFlags.includeDirs.emplace_back(resolveIncludeDir(depRoot),
-                                           /*isSystem=*/false);
-
   const fs::path depManifestPath = depRoot / Manifest::FILE_NAME;
   rs_ensure(fs::exists(depManifestPath), "missing `{}` in path dependency {}",
             Manifest::FILE_NAME, depRoot.string());
   const Manifest depManifest =
       rs_try(Manifest::tryParse(depManifestPath, false));
 
+  if (!suppressDepDiag) {
+    Diag::info("Building", "{} ({})", depManifest.package.name,
+               depRoot.string());
+  }
+
+  Builder depBuilder(depRoot, buildProfile);
+  ScheduleOptions depOptions;
+  depOptions.includeDevDeps = includeDevDeps;
+  depOptions.suppressAnalysisLog = true;
+  depOptions.suppressFinishLog = true;
+  depOptions.suppressDepDiag = suppressDepDiag;
+  rs_try(depBuilder.schedule(depOptions));
+  rs_try(depBuilder.build());
+
+  const BuildGraph& depGraph = depBuilder.graph();
+  const fs::path depOutDir = depGraph.outBasePath();
+  const fs::path libPath = depOutDir / depGraph.libraryName();
+
+  pathOpts.cFlags.includeDirs.emplace_back(resolveIncludeDir(depRoot),
+                                           /*isSystem=*/false);
+
   std::vector<CompilerOpts> nestedDeps;
-  rs_try(installDependencies(depManifest, includeDevDeps, seenDeps, visited,
-                             nestedDeps));
+  rs_try(installDependencies(depManifest, buildProfile, includeDevDeps,
+                             suppressDepDiag, seenDeps, visited, nestedDeps));
   for (const CompilerOpts& opts : nestedDeps) {
     pathOpts.merge(opts);
   }
 
+  const bool libBuilt = fs::exists(libPath);
+  if (depGraph.hasLibraryTarget()) {
+    rs_ensure(libBuilt, "expected `{}` to be built for dependency {}",
+              libPath.string(), depManifest.package.name);
+  }
+
+  if (libBuilt) {
+    pathOpts.ldFlags.libDirs.emplace(pathOpts.ldFlags.libDirs.begin(),
+                                     libPath.parent_path());
+
+    std::string libName = libPath.stem().string();
+    if (libName.starts_with("lib")) {
+      libName.erase(0, 3);
+    }
+    pathOpts.ldFlags.libs.emplace(pathOpts.ldFlags.libs.begin(),
+                                  std::move(libName));
+  }
+
   installed.emplace_back(std::move(pathOpts));
   return rs::Ok();
 }
@@ -201,7 +241,8 @@ rememberDep(const Manifest& manifest, const Dependency& dep,
 }
 
 static rs::Result<void>
-installDependencies(const Manifest& manifest, const bool includeDevDeps,
+installDependencies(const Manifest& manifest, const BuildProfile& buildProfile,
+                    const bool includeDevDeps, const bool suppressDepDiag,
                     std::unordered_map<std::string, DepKey>& seenDeps,
                     std::unordered_set<fs::path>& visited,
                     std::vector<CompilerOpts>& installed) {
@@ -219,7 +260,8 @@ installDependencies(const Manifest& manifest, const bool includeDevDeps,
                     rs_try(Manifest::tryParse(depManifestPath, false));
 
                 std::vector<CompilerOpts> nestedDeps;
-                rs_try(installDependencies(depManifest, includeDevDeps,
+                rs_try(installDependencies(depManifest, buildProfile,
+                                           includeDevDeps, suppressDepDiag,
                                            seenDeps, visited, nestedDeps));
                 for (const CompilerOpts& opts : nestedDeps) {
                   depOpts.merge(opts);
@@ -234,7 +276,8 @@ installDependencies(const Manifest& manifest, const bool includeDevDeps,
               return rs::Ok();
             },
             [&](const PathDependency& pathDep) -> rs::Result<void> {
-              return installPathDependency(manifest, pathDep, includeDevDeps,
+              return installPathDependency(manifest, pathDep, buildProfile,
+                                           includeDevDeps, suppressDepDiag,
                                            seenDeps, visited, installed);
             },
         },
@@ -663,12 +706,14 @@ rs::Result<fs::path> Manifest::findPath(fs::path candidateDir) noexcept {
 }
 
 rs::Result<std::vector<CompilerOpts>>
-Manifest::installDeps(const bool includeDevDeps) const {
+Manifest::installDeps(const bool includeDevDeps,
+                      const BuildProfile& buildProfile,
+                      const bool suppressDepDiag) const {
   std::unordered_map<std::string, DepKey> seenDeps;
   std::unordered_set<fs::path> visited;
   std::vector<CompilerOpts> installed;
-  rs_try(
-      installDependencies(*this, includeDevDeps, seenDeps, visited, installed));
+  rs_try(installDependencies(*this, buildProfile, includeDevDeps,
+                             suppressDepDiag, seenDeps, visited, installed));
   return rs::Ok(installed);
 }
 

semver/cabin.toml

@@ -0,0 +1,10 @@
+[package]
+name = "semver"
+edition = "23"
+version = "0.1.0"
+
+[dependencies]
+rs-cpp = { git = "https://github.com/ken-matsui/rs-cpp.git", branch = "main" }
+
+[profile]
+cxxflags = ["-pedantic-errors", "-Wall", "-Wextra", "-Wpedantic", "-fno-rtti"]