[libclang] Add API to generate a reproducer for the explicitly-built modules.

Capturing a single `-cc1` command to reproduce a bug with the explicitly-built
modules isn't sufficient, we need to know what .pcm files were built and how.

rdar://59743925

Author: vsapsai

clang/include/clang-c/Dependencies.h

@@ -325,6 +325,11 @@ CINDEX_LINKAGE void
     clang_experimental_DependencyScannerWorkerScanSettings_dispose(
         CXDependencyScannerWorkerScanSettings);
 
+CINDEX_LINKAGE enum CXErrorCode
+clang_experimental_DependencyScanner_generateReproducer(
+    int argc, const char *const *argv, const char *WorkingDirectory,
+    CXString *messageOut);
+
 /**
  * Produces the dependency graph for a particular compiler invocation.
  *

clang/test/Modules/reproducer-with-module-dependencies.c

@@ -0,0 +1,32 @@
+// Test generating a reproducer for a modular build where required modules are
+// built explicitly as separate steps.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+//
+// RUN: c-index-test core -gen-deps-reproducer -working-dir %t \
+// RUN:   -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \
+// RUN:      -fmodules -fmodules-cache-path=%t | FileCheck %t/reproducer.c
+
+// Test a failed attempt at generating a reproducer.
+// RUN: not c-index-test core -gen-deps-reproducer -working-dir %t \
+// RUN:   -- clang-executable -c %t/failed-reproducer.c -o %t/reproducer.o \
+// RUN:      -fmodules -fmodules-cache-path=%t 2>&1 | FileCheck %t/failed-reproducer.c
+
+//--- modular-header.h
+void fn_in_modular_header(void);
+
+//--- module.modulemap
+module Test { header "modular-header.h" export * }
+
+//--- reproducer.c
+// CHECK: Sources and associated run script(s) are located at:
+#include "modular-header.h"
+
+void test(void) {
+  fn_in_modular_header();
+}
+
+//--- failed-reproducer.c
+// CHECK: fatal error: 'non-existing-header.h' file not found
+#include "non-existing-header.h"

clang/tools/c-index-test/core_main.cpp

@@ -59,6 +59,7 @@ enum class ActionType {
   AggregateAsJSON,
   ScanDeps,
   ScanDepsByModuleName,
+  GenerateDepsReproducer,
   UploadCachedJob,
   MaterializeCachedJob,
   ReplayCachedJob,
@@ -87,6 +88,8 @@ Action(cl::desc("Action:"), cl::init(ActionType::None),
                      "Get file dependencies"),
           clEnumValN(ActionType::ScanDepsByModuleName, "scan-deps-by-mod-name",
                      "Get file dependencies by module name alone"),
+          clEnumValN(ActionType::GenerateDepsReproducer, "gen-deps-reproducer",
+                     "Generate a reproducer for the file"),
           clEnumValN(ActionType::UploadCachedJob, "upload-cached-job",
                      "Upload cached compilation data to upstream CAS"),
           clEnumValN(ActionType::MaterializeCachedJob, "materialize-cached-job",
@@ -921,6 +924,23 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
   return 1;
 }
 
+static int generateDepsReproducer(ArrayRef<const char *> Args,
+                                  std::string WorkingDirectory) {
+  CXString MessageString;
+  auto DisposeMessageString = llvm::make_scope_exit([&]() {
+    clang_disposeString(MessageString);
+  });
+  CXErrorCode ExitCode =
+      clang_experimental_DependencyScanner_generateReproducer(
+          Args.size(), Args.data(), WorkingDirectory.c_str(), &MessageString);
+  if (ExitCode == CXError_Success) {
+    llvm::outs() << clang_getCString(MessageString) << "\n";
+  } else {
+    llvm::errs() << "error: " << clang_getCString(MessageString) << "\n";
+  }
+  return (ExitCode == CXError_Success) ? 0 : 1;
+}
+
 static int uploadCachedJob(std::string CacheKey, CXCASDatabases DBs) {
   CXError Err = nullptr;
   CXCASCachedCompilation CComp = clang_experimental_cas_getCachedCompilation(
@@ -1546,6 +1566,14 @@ int indextest_core_main(int argc, const char **argv) {
                     options::OutputDir, DBs, options::ModuleName);
   }
 
+  if (options::Action == ActionType::GenerateDepsReproducer) {
+    if (options::WorkingDir.empty()) {
+      errs() << "error: missing -working-dir\n";
+      return 1;
+    }
+    return generateDepsReproducer(CompArgs, options::WorkingDir);
+  }
+
   if (options::Action == ActionType::UploadCachedJob) {
     if (options::InputFiles.empty()) {
       errs() << "error: missing cache key\n";

clang/tools/libclang/CDependencies.cpp

@@ -317,6 +317,132 @@ void clang_experimental_DependencyScannerWorkerScanSettings_dispose(
   delete unwrap(Settings);
 }
 
+namespace {
+// Helper class to capture a returnable error code and to return a formatted
+// message in a provided CXString pointer.
+class MessageEmitter {
+  const CXErrorCode ErrorCode;
+  CXString *OutputString;
+  std::string Buffer;
+  llvm::raw_string_ostream Stream;
+
+public:
+  MessageEmitter(CXErrorCode Code, CXString *Output)
+      : ErrorCode(Code), OutputString(Output), Stream(Buffer) {}
+  ~MessageEmitter() {
+    if (OutputString)
+      *OutputString = clang::cxstring::createDup(Buffer.c_str());
+  }
+
+  operator CXErrorCode() const { return ErrorCode; }
+
+  template <typename T> MessageEmitter &operator<<(const T &t) {
+    Stream << t;
+    return *this;
+  }
+};
+} // end anonymous namespace
+
+enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer(
+    int argc, const char *const *argv, const char *WorkingDirectory,
+    CXString *messageOut) {
+  auto report = [messageOut](CXErrorCode errorCode) -> MessageEmitter {
+    return MessageEmitter(errorCode, messageOut);
+  };
+  auto reportFailure = [&report]() -> MessageEmitter {
+    return report(CXError_Failure);
+  };
+
+  if (argc < 2 || !argv)
+    return report(CXError_InvalidArguments) << "missing compilation command";
+  if (!WorkingDirectory)
+    return report(CXError_InvalidArguments) << "missing working directory";
+
+  CASOptions CASOpts;
+  IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  DependencyScanningService DepsService(
+      ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Full,
+      CASOpts, /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS);
+  DependencyScanningTool DepsTool(DepsService);
+
+  llvm::SmallString<128> ReproScriptPath;
+  int ScriptFD;
+  if (auto EC = llvm::sys::fs::createTemporaryFile("reproducer", "sh", ScriptFD,
+                                                   ReproScriptPath)) {
+    return reportFailure() << "failed to create a reproducer script file";
+  }
+  SmallString<128> FileCachePath = ReproScriptPath;
+  llvm::sys::path::replace_extension(FileCachePath, ".cache");
+
+  std::string FileCacheName = llvm::sys::path::filename(FileCachePath).str();
+  auto LookupOutput = [&FileCacheName](const ModuleDeps &MD,
+                                       ModuleOutputKind MOK) -> std::string {
+    if (MOK != ModuleOutputKind::ModuleFile)
+      return "";
+    return FileCacheName + "/explicitly-built-modules/" +
+           MD.ID.ModuleName + "-" + MD.ID.ContextHash + ".pcm";
+  };
+
+  std::vector<std::string> Compilation{argv, argv + argc};
+  llvm::DenseSet<ModuleID> AlreadySeen;
+  auto TUDepsOrErr = DepsTool.getTranslationUnitDependencies(
+      Compilation, WorkingDirectory, AlreadySeen, std::move(LookupOutput));
+  if (!TUDepsOrErr)
+    return reportFailure() << "failed to generate a reproducer\n"
+                           << toString(TUDepsOrErr.takeError());
+
+  TranslationUnitDeps TU = *TUDepsOrErr;
+  llvm::raw_fd_ostream ScriptOS(ScriptFD, /*shouldClose=*/true);
+  ScriptOS << "# Original command:\n#";
+  for (StringRef cliArg : Compilation) {
+    ScriptOS << ' ' << cliArg;
+  }
+  ScriptOS << "\n\n";
+
+  ScriptOS << "# Dependencies:\n";
+  std::string ReproExecutable = std::string(argv[0]);
+  auto PrintArguments = [&ReproExecutable,
+                         &FileCacheName](llvm::raw_fd_ostream &OS,
+                                         ArrayRef<std::string> Arguments) {
+    OS << ReproExecutable;
+    for (int I = 0, E = Arguments.size(); I < E; ++I)
+      OS << ' ' << Arguments[I];
+    OS << " -ivfsoverlay \"" << FileCacheName << "/vfs/vfs.yaml\"";
+    OS << '\n';
+  };
+  for (ModuleDeps &dep : TU.ModuleGraph)
+    PrintArguments(ScriptOS, dep.getBuildArguments());
+  ScriptOS << "\n# Translation unit:\n";
+  for (const Command &buildCommand : TU.Commands)
+    PrintArguments(ScriptOS, buildCommand.Arguments);
+
+  SmallString<128> VFSCachePath = FileCachePath;
+  llvm::sys::path::append(VFSCachePath, "vfs");
+  std::string VFSCachePathStr = VFSCachePath.str().str();
+  llvm::FileCollector fileCollector(VFSCachePathStr,
+                                    /*OverlayRoot=*/VFSCachePathStr);
+  for (const auto &fileDep : TU.FileDeps) {
+    fileCollector.addFile(fileDep);
+  }
+  for (ModuleDeps &dep : TU.ModuleGraph) {
+    dep.forEachFileDep([&fileCollector](StringRef fileDep) {
+      fileCollector.addFile(fileDep);
+    });
+  }
+  if (fileCollector.copyFiles(/*StopOnError=*/true))
+    return reportFailure()
+           << "failed to copy the files used for the compilation";
+  SmallString<128> VFSOverlayPath = VFSCachePath;
+  llvm::sys::path::append(VFSOverlayPath, "vfs.yaml");
+  if (fileCollector.writeMapping(VFSOverlayPath))
+    return reportFailure() << "failed to write a VFS overlay mapping";
+
+  return report(CXError_Success)
+         << "Created a reproducer. Sources and associated run script(s) are "
+            "located at:\n  "
+         << FileCachePath << "\n  " << ReproScriptPath;
+}
+
 enum CXErrorCode clang_experimental_DependencyScannerWorker_getDepGraph(
     CXDependencyScannerWorker W,
     CXDependencyScannerWorkerScanSettings CXSettings, CXDepGraph *Out) {

clang/tools/libclang/libclang.map

@@ -496,6 +496,7 @@ LLVM_16 {
     clang_experimental_cas_replayCompilation;
     clang_experimental_cas_ReplayResult_dispose;
     clang_experimental_cas_ReplayResult_getStderr;
+    clang_experimental_DependencyScanner_generateReproducer;
     clang_experimental_DependencyScannerService_create_v1;
     clang_experimental_DependencyScannerServiceOptions_create;
     clang_experimental_DependencyScannerServiceOptions_dispose;