[clang][deps] Handle response files in dep scanner

Extract the code the driver uses to expand response files and reuse it
in the dependency scanner.

rdar://106155880

Differential Revision: https://reviews.llvm.org/D145838

(cherry picked from commit fcab930)

 Conflicts:
	clang/include/clang/Basic/DiagnosticDriverKinds.td
	clang/include/clang/Driver/Driver.h
	clang/lib/Driver/Driver.cpp
	clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
	clang/tools/driver/driver.cpp

Author: benlangmuir

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -704,4 +704,7 @@ def warn_drv_sarif_format_unstable : Warning<
 
 def err_drv_riscv_unsupported_with_linker_relaxation : Error<
   "%0 is unsupported with RISC-V linker relaxation (-mrelax)">;
+
+def err_drv_expand_response_file : Error<
+  "failed to expand response file: %0">;
 }

clang/include/clang/Driver/Driver.h

@@ -788,6 +788,16 @@ bool IsClangCL(StringRef DriverMode);
 /// Checks whether the value produced by getDriverMode is for 'cache' mode.
 bool isClangCache(StringRef DriverMode);
 
+/// Expand response files from a clang driver or cc1 invocation.
+///
+/// \param Args The arguments that will be expanded.
+/// \param ClangCLMode Whether clang is in CL mode.
+/// \param Alloc Allocator for new arguments.
+/// \param FS Filesystem to use when expanding files.
+llvm::Error expandResponseFiles(SmallVectorImpl<const char *> &Args,
+                                bool ClangCLMode, llvm::BumpPtrAllocator &Alloc,
+                                llvm::vfs::FileSystem *FS = nullptr);
+
 } // end namespace driver
 } // end namespace clang
 

clang/lib/Driver/Driver.cpp

@@ -6340,3 +6340,58 @@ bool driver::IsClangCL(StringRef DriverMode) { return DriverMode.equals("cl"); }
 bool driver::isClangCache(StringRef DriverMode) {
   return DriverMode == "cache";
 }
+
+llvm::Error driver::expandResponseFiles(SmallVectorImpl<const char *> &Args,
+                                        bool ClangCLMode,
+                                        llvm::BumpPtrAllocator &Alloc,
+                                        llvm::vfs::FileSystem *FS) {
+  // Parse response files using the GNU syntax, unless we're in CL mode. There
+  // are two ways to put clang in CL compatibility mode: ProgName is either
+  // clang-cl or cl, or --driver-mode=cl is on the command line. The normal
+  // command line parsing can't happen until after response file parsing, so we
+  // have to manually search for a --driver-mode=cl argument the hard way.
+  // Finally, our -cc1 tools don't care which tokenization mode we use because
+  // response files written by clang will tokenize the same way in either mode.
+  enum { Default, POSIX, Windows } RSPQuoting = Default;
+  for (const char *F : Args) {
+    if (strcmp(F, "--rsp-quoting=posix") == 0)
+      RSPQuoting = POSIX;
+    else if (strcmp(F, "--rsp-quoting=windows") == 0)
+      RSPQuoting = Windows;
+  }
+
+  // Determines whether we want nullptr markers in Args to indicate response
+  // files end-of-lines. We only use this for the /LINK driver argument with
+  // clang-cl.exe on Windows.
+  bool MarkEOLs = ClangCLMode;
+
+  llvm::cl::TokenizerCallback Tokenizer;
+  if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
+    Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
+  else
+    Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
+
+  if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1"))
+    MarkEOLs = false;
+
+  llvm::cl::ExpansionContext ECtx(Alloc, Tokenizer);
+  ECtx.setMarkEOLs(MarkEOLs);
+  if (FS)
+    ECtx.setVFS(FS);
+
+  // Note: on this branch the driver expects us to *not* propagate any errors.
+  ECtx.expandResponseFiles(Args);
+
+  // If -cc1 came from a response file, remove the EOL sentinels.
+  auto FirstArg = llvm::find_if(llvm::drop_begin(Args),
+                                [](const char *A) { return A != nullptr; });
+  if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) {
+    // If -cc1 came from a response file, remove the EOL sentinels.
+    if (MarkEOLs) {
+      auto newEnd = std::remove(Args.begin(), Args.end(), nullptr);
+      Args.resize(newEnd - Args.begin());
+    }
+  }
+
+  return llvm::Error::success();
+}

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -8,6 +8,7 @@
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
 #include "clang/Basic/DiagnosticCAS.h"
+#include "clang/Basic/DiagnosticDriver.h"
 #include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
 #include "clang/Driver/Compilation.h"
@@ -27,6 +28,8 @@
 #include "clang/Tooling/Tooling.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/PrefixMapper.h"
 
@@ -619,16 +622,29 @@ llvm::Error DependencyScanningWorker::computeDependencies(
 }
 
 static bool forEachDriverJob(
-    ArrayRef<std::string> Args, DiagnosticsEngine &Diags, FileManager &FM,
+    ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, FileManager &FM,
     llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
+  SmallVector<const char *, 256> Argv;
+  Argv.reserve(ArgStrs.size());
+  for (const std::string &Arg : ArgStrs)
+    Argv.push_back(Arg.c_str());
+
+  llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem();
+
   std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
-      Args[0], llvm::sys::getDefaultTargetTriple(), Diags,
-      "clang LLVM compiler", &FM.getVirtualFileSystem());
+      Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
+      "clang LLVM compiler", FS);
   Driver->setTitle("clang_based_tool");
 
-  std::vector<const char *> Argv;
-  for (const std::string &Arg : Args)
-    Argv.push_back(Arg.c_str());
+  llvm::BumpPtrAllocator Alloc;
+  bool CLMode = driver::IsClangCL(
+      driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
+
+  if (llvm::Error E = driver::expandResponseFiles(Argv, CLMode, Alloc, FS)) {
+    Diags.Report(diag::err_drv_expand_response_file)
+        << llvm::toString(std::move(E));
+    return false;
+  }
 
   const std::unique_ptr<driver::Compilation> Compilation(
       Driver->BuildCompilation(llvm::makeArrayRef(Argv)));

clang/test/ClangScanDeps/response-file.c

@@ -0,0 +1,41 @@
+// Check that the scanner can handle a response file input.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+
+// RUN: clang-scan-deps -format experimental-full -compilation-database %t/cdb.json > %t/deps.json
+
+// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s
+
+// CHECK:      "command-line": [
+// CHECK:        "-fsyntax-only"
+// CHECK:        "-x"
+// CHECK-NEXT:   "c"
+// CHECK:        "tu.c"
+// CHECK:        "-I"
+// CHECK-NEXT:   "include"
+// CHECK:      ],
+// CHECK:      "file-deps": [
+// CHECK-NEXT:   "[[PREFIX]]/tu.c"
+// CHECK-NEXT:   "[[PREFIX]]/include/header.h"
+// CHECK-NEXT: ]
+
+//--- cdb.json.template
+[{
+  "file": "DIR/t.c",
+  "directory": "DIR",
+  "command": "clang @DIR/args.txt"
+}]
+
+//--- args.txt
+@args_nested.txt
+-fsyntax-only tu.c
+
+//--- args_nested.txt
+-I include
+
+//--- include/header.h
+
+//--- tu.c
+#include "header.h"

clang/tools/driver/driver.cpp

@@ -411,50 +411,16 @@ int clang_main(int Argc, char **Argv) {
       return *ExitCode;
   }
 
-  // Parse response files using the GNU syntax, unless we're in CL mode. There
-  // are two ways to put clang in CL compatibility mode: Args[0] is either
-  // clang-cl or cl, or --driver-mode=cl is on the command line. The normal
-  // command line parsing can't happen until after response file parsing, so we
-  // have to manually search for a --driver-mode=cl argument the hard way.
-  // Finally, our -cc1 tools don't care which tokenization mode we use because
-  // response files written by clang will tokenize the same way in either mode.
   bool ClangCLMode = IsClangCL(DriverMode);
-  enum { Default, POSIX, Windows } RSPQuoting = Default;
-  for (const char *F : Args) {
-    if (strcmp(F, "--rsp-quoting=posix") == 0)
-      RSPQuoting = POSIX;
-    else if (strcmp(F, "--rsp-quoting=windows") == 0)
-      RSPQuoting = Windows;
+
+  if (llvm::Error Err = expandResponseFiles(Args, ClangCLMode, A)) {
+    llvm::errs() << toString(std::move(Err)) << '\n';
+    return 1;
   }
 
-  // Determines whether we want nullptr markers in Args to indicate response
-  // files end-of-lines. We only use this for the /LINK driver argument with
-  // clang-cl.exe on Windows.
-  bool MarkEOLs = ClangCLMode;
-
-  llvm::cl::TokenizerCallback Tokenizer;
-  if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
-    Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
-  else
-    Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
-
-  if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1"))
-    MarkEOLs = false;
-  llvm::cl::ExpansionContext ECtx(A, Tokenizer);
-  ECtx.setMarkEOLs(MarkEOLs).expandResponseFiles(Args);
-
-  // Handle -cc1 integrated tools, even if -cc1 was expanded from a response
-  // file.
-  auto FirstArg = llvm::find_if(llvm::drop_begin(Args),
-                                [](const char *A) { return A != nullptr; });
-  if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) {
-    // If -cc1 came from a response file, remove the EOL sentinels.
-    if (MarkEOLs) {
-      auto newEnd = std::remove(Args.begin(), Args.end(), nullptr);
-      Args.resize(newEnd - Args.begin());
-    }
+  // Handle -cc1 integrated tools.
+  if (Args.size() >= 2 && StringRef(Args[1]).startswith("-cc1"))
     return ExecuteCC1Tool(Args);
-  }
 
   // Handle options that need handling before the real command line parsing in
   // Driver::BuildCompilation()