diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 2403cbbb652dd..18ad7bf292f1e 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -946,6 +946,12 @@ class CompilerInstance : public ModuleLoader { DependencyCollectors.push_back(std::move(Listener)); } + void clearDependencyCollectors() { DependencyCollectors.clear(); } + + std::vector> &getDependencyCollectors() { + return DependencyCollectors; + } + void setExternalSemaSource(IntrusiveRefCntPtr ESS); ModuleCache &getModuleCache() const { return *ModCache; } diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index 73308c004bd23..87a9f0d4cb06c 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -320,15 +320,6 @@ class PrintPreprocessedAction : public PreprocessorFrontendAction { bool hasPCHSupport() const override { return true; } }; -class GetDependenciesByModuleNameAction : public PreprocessOnlyAction { - StringRef ModuleName; - void ExecuteAction() override; - -public: - GetDependenciesByModuleNameAction(StringRef ModuleName) - : ModuleName(ModuleName) {} -}; - //===----------------------------------------------------------------------===// // HLSL Specific Actions //===----------------------------------------------------------------------===// diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 39754847a93e4..953902b13783f 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -1327,6 +1327,7 @@ class Preprocessor { std::move(Callbacks)); Callbacks = std::move(C); } + void removePPCallbacks() { Callbacks.reset(); } /// \} /// Get the number of tokens processed so far. diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index f222ded8a966a..ed562f46cfdaa 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -154,11 +154,52 @@ class DependencyScanningTool { /// Given a compilation context specified via the Clang driver command-line, /// gather modular dependencies of module with the given name, and return the /// information needed for explicit build. + /// TODO: this method should be removed as soon as Swift and our C-APIs adopt + /// CompilerInstanceWithContext. We are keeping it here so that it is easier + /// to coordinate with Swift and C-API changes. llvm::Expected getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::DenseSet &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput); + /// The following three methods provide a new interface to perform + /// by name dependency scan. The new interface's intention is to improve + /// dependency scanning performance when a sequence of name is looked up + /// with the same current working directory and the command line. + + /// @brief Initializing the context and the compiler instance. + /// This method must be called before calling + /// computeDependenciesByNameWithContext. + /// @param CWD The current working directory used during the scan. + /// @param CommandLine The commandline used for the scan. + /// @return Error if the initializaiton fails. + llvm::Error initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine); + + /// @brief Computes the dependeny for the module named ModuleName. + /// @param ModuleName The name of the module for which this method computes + ///. dependencies. + /// @param AlreadySeen This stores modules which have previously been + /// reported. Use the same instance for all calls to this + /// function for a single \c DependencyScanningTool in a + /// single build. Note that this parameter is not part of + /// the context because it can be shared across different + /// worker threads and each worker thread may update it. + /// @param LookupModuleOutput This function is called to fill in + /// "-fmodule-file=", "-o" and other output + /// arguments for dependencies. + /// @return An instance of \c TranslationUnitDeps if the scan is successful. + /// Otherwise it returns an error. + llvm::Expected computeDependenciesByNameWithContext( + StringRef ModuleName, const llvm::DenseSet &AlreadySeen, + LookupModuleOutputCallback LookupModuleOutput); + + /// @brief This method finializes the compiler instance. It finalizes the + /// diagnostics and deletes the compiler instance. Call this method + /// once all names for a same commandline are scanned. + /// @return Error if an error occured during finalization. + llvm::Error finalizeCompilerInstanceWithContext(); + llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); } private: diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 6060e4b43312e..e2c353a254bf3 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -29,6 +29,7 @@ namespace tooling { namespace dependencies { class DependencyScanningWorkerFilesystem; +class CompilerInstanceWithContext; /// A command-line tool invocation that is part of building a TU. /// @@ -89,6 +90,8 @@ class DependencyScanningWorker { DependencyScanningWorker(DependencyScanningService &Service, llvm::IntrusiveRefCntPtr FS); + ~DependencyScanningWorker(); + /// Run the dependency scanning tool for a given clang driver command-line, /// and report the discovered dependencies to the provided consumer. If /// TUBuffer is not nullopt, it is used as TU input for the dependency @@ -103,18 +106,6 @@ class DependencyScanningWorker { DiagnosticConsumer &DiagConsumer, std::optional TUBuffer = std::nullopt); - /// Run the dependency scanning tool for a given clang driver command-line - /// for a specific module. - /// - /// \returns false if clang errors occurred (with diagnostics reported to - /// \c DiagConsumer), true otherwise. - bool computeDependencies(StringRef WorkingDirectory, - const std::vector &CommandLine, - DependencyConsumer &DepConsumer, - DependencyActionController &Controller, - DiagnosticConsumer &DiagConsumer, - StringRef ModuleName); - /// Run the dependency scanning tool for a given clang driver command-line /// for a specific translation unit via file system or memory buffer. /// @@ -125,16 +116,46 @@ class DependencyScanningWorker { DependencyConsumer &Consumer, DependencyActionController &Controller, std::optional TUBuffer = std::nullopt); - /// Run the dependency scanning tool for a given clang driver command-line - /// for a specific module. - /// - /// \returns A \c StringError with the diagnostic output if clang errors - /// occurred, success otherwise. - llvm::Error computeDependencies(StringRef WorkingDirectory, - const std::vector &CommandLine, - DependencyConsumer &Consumer, - DependencyActionController &Controller, - StringRef ModuleName); + /// The three method below implements a new interface for by name + /// dependency scanning. They together enable the dependency scanning worker + /// to more effectively perform scanning for a sequence of modules + /// by name when the CWD and CommandLine do not change across the queries. + + /// @brief Initializing the context and the compiler instance. + /// @param CWD The current working directory used during the scan. + /// @param CommandLine The commandline used for the scan. + /// @return Error if the initializaiton fails. + llvm::Error initializeCompilerInstanceWithContextOrError( + StringRef CWD, const std::vector &CommandLine); + + /// @brief Performaces dependency scanning for the module whose name is + /// specified. + /// @param ModuleName The name of the module whose dependency will be + /// scanned. + /// @param Consumer The dependency consumer that stores the results. + /// @param Controller The controller for the dependency scanning action. + /// @return Error if the scanner incurs errors. + llvm::Error computeDependenciesByNameWithContextOrError( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller); + + /// @brief Finalizes the diagnostics engine and deletes the compiler instance. + /// @return Error if errors occur during finalization. + llvm::Error finalizeCompilerInstanceWithContextOrError(); + + /// The three methods below provides the same functionality as the + /// three methods above. Instead of returning `llvm::Error`s, these + /// three methods return a flag to indicate if the call is successful. + /// The initialization function asks the client for a DiagnosticsConsumer + /// that it direct the diagnostics to. + bool initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine, + DiagnosticConsumer *DC = nullptr); + bool + computeDependenciesByNameWithContext(StringRef ModuleName, + DependencyConsumer &Consumer, + DependencyActionController &Controller); + bool finalizeCompilerInstance(); llvm::vfs::FileSystem &getVFS() const { return *BaseFS; } @@ -151,14 +172,16 @@ class DependencyScanningWorker { /// (passed in the constructor). llvm::IntrusiveRefCntPtr DepFS; + friend CompilerInstanceWithContext; + std::unique_ptr CIWithContext; + /// Private helper functions. bool scanDependencies(StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, DiagnosticConsumer &DC, - llvm::IntrusiveRefCntPtr FS, - std::optional ModuleName); + llvm::IntrusiveRefCntPtr FS); }; } // end namespace dependencies diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index 4136cb73f7043..b0a91b60ff6da 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -288,6 +288,8 @@ class ModuleDepCollector final : public DependencyCollector { void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; + PPCallbacks *getPPCallbacks() { return CollectorPPPtr; } + /// Apply any changes implied by the discovered dependencies to the given /// invocation, (e.g. disable implicit modules, add explicit module paths). void applyDiscoveredDependencies(CompilerInvocation &CI); @@ -339,6 +341,11 @@ class ModuleDepCollector final : public DependencyCollector { std::optional ProvidedStdCXXModule; std::vector RequiredStdCXXModules; + /// A pointer to the preprocessor callback so we can invoke it directly + /// if needed. The callback is created and added to a Preprocessor instance by + /// attachToPreprocessor and the Preprocessor instance owns it. + ModuleDepCollectorPP *CollectorPPPtr = nullptr; + /// Checks whether the module is known as being prebuilt. bool isPrebuiltModule(const Module *M); diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index d7d56b8166350..3595bbc6c9b9e 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -1233,20 +1233,6 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() { llvm::outs()); } -void GetDependenciesByModuleNameAction::ExecuteAction() { - CompilerInstance &CI = getCompilerInstance(); - Preprocessor &PP = CI.getPreprocessor(); - SourceManager &SM = PP.getSourceManager(); - FileID MainFileID = SM.getMainFileID(); - SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID); - SmallVector Path; - IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName); - Path.emplace_back(FileStart, ModuleID); - auto ModResult = CI.loadModule(FileStart, Path, Module::Hidden, false); - PPCallbacks *CB = PP.getPPCallbacks(); - CB->moduleImport(SourceLocation(), Path, ModResult); -} - //===----------------------------------------------------------------------===// // HLSL Specific Actions //===----------------------------------------------------------------------===// diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp index 42f52d0ff6241..629780e18fd4d 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp @@ -1,4 +1,4 @@ -//===- DependencyScanner.cpp - Performs module dependency scanning --------===// +//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -12,6 +12,7 @@ #include "clang/Driver/Driver.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/TargetParser/Host.h" using namespace clang; @@ -456,7 +457,8 @@ initVFSForTUBuferScanning(IntrusiveRefCntPtr BaseFS, return std::make_pair(ModifiedFS, ModifiedCommandLine); } -std::pair, std::vector> +std::pair, + std::vector> initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, ArrayRef CommandLine, StringRef WorkingDirectory, StringRef ModuleName) { @@ -588,7 +590,7 @@ computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, } std::unique_ptr -takeDependencyOutputOptionsFrom(CompilerInstance &ScanInstance) { +takeAndUpdateDependencyOutputOptionsFrom(CompilerInstance &ScanInstance) { // This function moves the existing dependency output options from the // invocation to the collector. The options in the invocation are reset, // which ensures that the compiler won't create new dependency collectors, @@ -675,7 +677,7 @@ bool DependencyScanningAction::runInvocation( if (!MaybePrebuiltModulesASTMap) return false; - auto DepOutputOpts = takeDependencyOutputOptionsFrom(ScanInstance); + auto DepOutputOpts = takeAndUpdateDependencyOutputOptionsFrom(ScanInstance); MDC = initializeScanInstanceDependencyCollector( ScanInstance, std::move(DepOutputOpts), WorkingDirectory, Consumer, @@ -686,8 +688,6 @@ bool DependencyScanningAction::runInvocation( if (Service.getFormat() == ScanningOutputFormat::P1689) Action = std::make_unique(); - else if (ModuleName) - Action = std::make_unique(*ModuleName); else Action = std::make_unique(); @@ -704,3 +704,178 @@ bool DependencyScanningAction::runInvocation( return Result; } + +bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) { + if (DC) { + DiagConsumer = DC; + } else { + DiagPrinterWithOS = + std::make_unique(CommandLine); + DiagConsumer = &DiagPrinterWithOS->DiagPrinter; + } + + DiagEngineWithCmdAndOpts = std::make_unique( + CommandLine, OverlayFS, *DiagConsumer); + + std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning( + Worker.BaseFS, CommandLine, CWD, "ScanningByName"); + + std::tie(Driver, Compilation) = buildCompilation( + CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS, Alloc); + + if (!Compilation) + return false; + + assert(Compilation->getJobs().size() && + "Must have a job list of non-zero size"); + const driver::Command &Command = *(Compilation->getJobs().begin()); + const auto &CommandArgs = Command.getArguments(); + size_t ArgSize = CommandArgs.size(); + assert(ArgSize >= 1 && "Cannot have a command with 0 args"); + const char *FirstArg = CommandArgs[0]; + assert(StringRef(FirstArg) == "-cc1" && "Requires a cc1 job."); + OriginalInvocation = std::make_unique(); + + if (!CompilerInvocation::CreateFromArgs( + *OriginalInvocation, Command.getArguments(), + *DiagEngineWithCmdAndOpts->DiagEngine, Command.getExecutable())) { + DiagEngineWithCmdAndOpts->DiagEngine->Report( + diag::err_fe_expected_compiler_job) + << llvm::join(CommandLine, " "); + return false; + } + + if (any(Worker.Service.getOptimizeArgs() & ScanningOptimizations::Macros)) + canonicalizeDefines(OriginalInvocation->getPreprocessorOpts()); + + // Create the CompilerInstance. + IntrusiveRefCntPtr ModCache = + makeInProcessModuleCache(Worker.Service.getModuleCacheEntries()); + CIPtr = std::make_unique( + std::make_shared(*OriginalInvocation), + Worker.PCHContainerOps, ModCache.get()); + auto &CI = *CIPtr; + + if (!initializeScanCompilerInstance( + CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(), + Worker.Service, Worker.DepFS)) + return false; + + llvm::SmallVector StableDirs = getInitialStableDirs(CI); + auto MaybePrebuiltModulesASTMap = + computePrebuiltModulesASTMap(CI, StableDirs); + if (!MaybePrebuiltModulesASTMap) + return false; + + PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap); + OutputOpts = takeAndUpdateDependencyOutputOptionsFrom(CI); + + // We do not create the target in initializeScanCompilerInstance because + // setting it here is unique for by-name lookups. We create the target only + // once here, and the information is reused for all computeDependencies calls. + // We do not need to call createTarget explicitly if we go through + // CompilerInstance::ExecuteAction to perform scanning. + CI.createTarget(); + + return true; +} + +bool CompilerInstanceWithContext::computeDependencies( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller) { + assert(CIPtr && "CIPtr must be initialized before calling this method"); + auto &CI = *CIPtr; + + // We create this cleanup object because computeDependencies may exit + // early with errors. + auto CleanUp = llvm::make_scope_exit([&]() { + CI.clearDependencyCollectors(); + // The preprocessor may not be created at the entry of this method, + // but it must have been created when this method returns, whether + // there are errors during scanning or not. + CI.getPreprocessor().removePPCallbacks(); + }); + + CI.clearDependencyCollectors(); + auto MDC = initializeScanInstanceDependencyCollector( + CI, std::make_unique(*OutputOpts), CWD, Consumer, + Worker.Service, + /* The MDC's constructor makes a copy of the OriginalInvocation, so + we can pass it in without worrying that it might be changed across + invocations of computeDependencies. */ + *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs); + + if (!SrcLocOffset) { + // When SrcLocOffset is zero, we are at the beginning of the fake source + // file. In this case, we call BeginSourceFile to initialize. + std::unique_ptr Action = + std::make_unique(); + auto InputFile = CI.getFrontendOpts().Inputs.begin(); + bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile); + assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed"); + (void)ActionBeginSucceeded; + } + + Preprocessor &PP = CI.getPreprocessor(); + SourceManager &SM = PP.getSourceManager(); + FileID MainFileID = SM.getMainFileID(); + SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID); + SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset); + PPCallbacks *CB = nullptr; + if (!SrcLocOffset) { + // We need to call EnterSourceFile when SrcLocOffset is zero to initialize + // the preprocessor. + bool PPFailed = PP.EnterSourceFile(MainFileID, nullptr, SourceLocation()); + assert(!PPFailed && "Preprocess must be able to enter the main file."); + (void)PPFailed; + CB = MDC->getPPCallbacks(); + } else { + // When SrcLocOffset is non-zero, the preprocessor has already been + // initialized through a previous call of computeDependencies. We want to + // preserve the PP's state, hence we do not call EnterSourceFile again. + MDC->attachToPreprocessor(PP); + CB = MDC->getPPCallbacks(); + + FileID PrevFID; + SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(IDLocation); + CB->LexedFileChanged(MainFileID, + PPChainedCallbacks::LexedFileChangeReason::EnterFile, + FileType, PrevFID, IDLocation); + } + + SrcLocOffset++; + SmallVector Path; + IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName); + Path.emplace_back(IDLocation, ModuleID); + auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false); + + assert(CB && "Must have PPCallbacks after module loading"); + CB->moduleImport(SourceLocation(), Path, ModResult); + // Note that we are calling the CB's EndOfMainFile function, which + // forwards the results to the dependency consumer. + // It does not indicate the end of processing the fake file. + CB->EndOfMainFile(); + + if (!ModResult) + return false; + + CompilerInvocation ModuleInvocation(*OriginalInvocation); + MDC->applyDiscoveredDependencies(ModuleInvocation); + Consumer.handleBuildCommand( + {CommandLine[0], ModuleInvocation.getCC1CommandLine()}); + + return true; +} + +bool CompilerInstanceWithContext::finalize() { + DiagConsumer->finish(); + return true; +} + +llvm::Error CompilerInstanceWithContext::handleReturnStatus(bool Success) { + assert(DiagPrinterWithOS && "Must use the default DiagnosticConsumer."); + return Success ? llvm::Error::success() + : llvm::make_error( + DiagPrinterWithOS->DiagnosticsOS.str(), + llvm::inconvertibleErrorCode()); +} diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h index 5657317565e8d..54166dabebb05 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h @@ -1,4 +1,4 @@ -//===- DependencyScanner.h - Performs module dependency scanning *- C++ -*-===// +//===- DependencyScannerImpl.h - Implements dependency scanning *- C++ -*--===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -23,6 +23,8 @@ class DiagnosticConsumer; namespace tooling { namespace dependencies { class DependencyScanningService; +class DependencyScanningWorker; + class DependencyConsumer; class DependencyActionController; class DependencyScanningWorkerFilesystem; @@ -35,8 +37,7 @@ class DependencyScanningAction { IntrusiveRefCntPtr DepFS, std::optional ModuleName = std::nullopt) : Service(Service), WorkingDirectory(WorkingDirectory), - Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)), - ModuleName(ModuleName) {} + Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)) {} bool runInvocation(std::unique_ptr Invocation, IntrusiveRefCntPtr FS, std::shared_ptr PCHContainerOps, @@ -66,7 +67,6 @@ class DependencyScanningAction { DependencyConsumer &Consumer; DependencyActionController &Controller; IntrusiveRefCntPtr DepFS; - std::optional ModuleName; std::optional ScanInstanceStorage; std::shared_ptr MDC; std::vector LastCC1Arguments; @@ -118,7 +118,8 @@ initVFSForTUBuferScanning(IntrusiveRefCntPtr BaseFS, StringRef WorkingDirectory, llvm::MemoryBufferRef TUBuffer); -std::pair, std::vector> +std::pair, + std::vector> initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, ArrayRef CommandLine, StringRef WorkingDirectory, StringRef ModuleName); @@ -137,7 +138,7 @@ computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, SmallVector &StableDirs); std::unique_ptr -takeDependencyOutputOptionsFrom(CompilerInstance &ScanInstance); +takeAndUpdateDependencyOutputOptionsFrom(CompilerInstance &ScanInstance); /// Create the dependency collector that will collect the produced /// dependencies. May return the created ModuleDepCollector depending @@ -150,6 +151,60 @@ std::shared_ptr initializeScanInstanceDependencyCollector( DependencyActionController &Controller, PrebuiltModulesAttrsMap PrebuiltModulesASTMap, llvm::SmallVector &StableDirs); + +class CompilerInstanceWithContext { + // Context + DependencyScanningWorker &Worker; + llvm::StringRef CWD; + std::vector CommandLine; + + // Context - file systems + llvm::IntrusiveRefCntPtr OverlayFS; + + // Context - Diagnostics engine. + std::unique_ptr DiagPrinterWithOS; + // DiagConsumer may points to DiagPrinterWithOS->DiagPrinter, or a custom + // DiagnosticConsumer passed in from initialize. + DiagnosticConsumer *DiagConsumer = nullptr; + std::unique_ptr DiagEngineWithCmdAndOpts; + + // Context - compiler invocation + // Compilation's command's arguments may be owned by Alloc when expanded from + // response files, so we need to keep Alloc alive in the context. + llvm::BumpPtrAllocator Alloc; + std::unique_ptr Driver; + std::unique_ptr Compilation; + std::unique_ptr OriginalInvocation; + + // Context - output options + std::unique_ptr OutputOpts; + + // Context - stable directory handling + llvm::SmallVector StableDirs; + PrebuiltModulesAttrsMap PrebuiltModuleASTMap; + + // Compiler Instance + std::unique_ptr CIPtr; + + // Source location offset. + int32_t SrcLocOffset = 0; + +public: + CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD, + const std::vector &CMD) + : Worker(Worker), CWD(CWD), CommandLine(CMD) {}; + + // The three methods below returns false when they fail, with the detail + // accumulated in DiagConsumer. + bool initialize(DiagnosticConsumer *DC); + bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller); + bool finalize(); + + // The method below turns the return status from the above methods + // into an llvm::Error using a default DiagnosticConsumer. + llvm::Error handleReturnStatus(bool Success); +}; } // namespace dependencies } // namespace tooling } // namespace clang diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 27734ffd0e20b..a1f2db7a471be 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -162,13 +162,45 @@ DependencyScanningTool::getModuleDependencies( LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); CallbackActionController Controller(LookupModuleOutput); - llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, - Controller, ModuleName); + if (auto Error = + Worker.initializeCompilerInstanceWithContextOrError(CWD, CommandLine)) + return std::move(Error); + + auto Result = Worker.computeDependenciesByNameWithContextOrError( + ModuleName, Consumer, Controller); + + if (auto Error = Worker.finalizeCompilerInstanceWithContextOrError()) + return std::move(Error); + if (Result) return std::move(Result); + return Consumer.takeTranslationUnitDeps(); } +llvm::Error DependencyScanningTool::initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine) { + return Worker.initializeCompilerInstanceWithContextOrError(CWD, CommandLine); +} + +llvm::Expected +DependencyScanningTool::computeDependenciesByNameWithContext( + StringRef ModuleName, const llvm::DenseSet &AlreadySeen, + LookupModuleOutputCallback LookupModuleOutput) { + FullDependencyConsumer Consumer(AlreadySeen); + CallbackActionController Controller(LookupModuleOutput); + llvm::Error Result = Worker.computeDependenciesByNameWithContextOrError( + ModuleName, Consumer, Controller); + if (Result) + return std::move(Result); + + return Consumer.takeTranslationUnitDeps(); +} + +llvm::Error DependencyScanningTool::finalizeCompilerInstanceWithContext() { + return Worker.finalizeCompilerInstanceWithContextOrError(); +} + TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { TranslationUnitDeps TU; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 0a1cf6b18b11c..dc408b10542c3 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -43,6 +43,9 @@ DependencyScanningWorker::DependencyScanningWorker( } } +DependencyScanningWorker::~DependencyScanningWorker() = default; +DependencyActionController::~DependencyActionController() = default; + llvm::Error DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, @@ -58,21 +61,6 @@ llvm::Error DependencyScanningWorker::computeDependencies( DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); } -llvm::Error DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - StringRef ModuleName) { - // Capture the emitted diagnostics and report them to the client - // in the case of a failure. - TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); - - if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DiagPrinterWithOS.DiagPrinter, ModuleName)) - return llvm::Error::success(); - return llvm::make_error( - DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); -} - static bool forEachDriverJob( ArrayRef ArgStrs, DiagnosticsEngine &Diags, IntrusiveRefCntPtr FS, @@ -113,11 +101,11 @@ static bool createAndRunToolInvocation( bool DependencyScanningWorker::scanDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr FS, - std::optional ModuleName) { + DiagnosticConsumer &DC, + llvm::IntrusiveRefCntPtr FS) { DignosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC); DependencyScanningAction Action(Service, WorkingDirectory, Consumer, - Controller, DepFS, ModuleName); + Controller, DepFS); bool Success = false; if (CommandLine[1] == "-cc1") { @@ -172,24 +160,51 @@ bool DependencyScanningWorker::computeDependencies( auto [FinalFS, FinalCommandLine] = initVFSForTUBuferScanning( BaseFS, CommandLine, WorkingDirectory, *TUBuffer); return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer, - Controller, DC, FinalFS, - /*ModuleName=*/std::nullopt); + Controller, DC, FinalFS); } else { BaseFS->setCurrentWorkingDirectory(WorkingDirectory); return scanDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DC, BaseFS, /*ModuleName=*/std::nullopt); + DC, BaseFS); } } -bool DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, StringRef ModuleName) { - auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning( - BaseFS, CommandLine, WorkingDirectory, ModuleName); +llvm::Error +DependencyScanningWorker::initializeCompilerInstanceWithContextOrError( + StringRef CWD, const std::vector &CommandLine) { + bool Success = initializeCompilerInstanceWithContext(CWD, CommandLine); + return CIWithContext->handleReturnStatus(Success); +} + +llvm::Error +DependencyScanningWorker::computeDependenciesByNameWithContextOrError( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller) { + bool Success = + computeDependenciesByNameWithContext(ModuleName, Consumer, Controller); + return CIWithContext->handleReturnStatus(Success); +} + +llvm::Error +DependencyScanningWorker::finalizeCompilerInstanceWithContextOrError() { + bool Success = finalizeCompilerInstance(); + return CIWithContext->handleReturnStatus(Success); +} - return scanDependencies(WorkingDirectory, ModifiedCommandLine, Consumer, - Controller, DC, OverlayFS, ModuleName); +bool DependencyScanningWorker::initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine, + DiagnosticConsumer *DC) { + CIWithContext = + std::make_unique(*this, CWD, CommandLine); + return CIWithContext->initialize(DC); } -DependencyActionController::~DependencyActionController() {} +bool DependencyScanningWorker::computeDependenciesByNameWithContext( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller) { + assert(CIWithContext && "CompilerInstance with context required!"); + return CIWithContext->computeDependencies(ModuleName, Consumer, Controller); +} + +bool DependencyScanningWorker::finalizeCompilerInstance() { + return CIWithContext->finalize(); +} diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index a117bec0d656e..e07a208748b77 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -965,7 +965,9 @@ ModuleDepCollector::ModuleDepCollector( makeCommonInvocationForModuleBuild(std::move(OriginalCI))) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { - PP.addPPCallbacks(std::make_unique(*this)); + auto CollectorPP = std::make_unique(*this); + CollectorPPPtr = CollectorPP.get(); + PP.addPPCallbacks(std::move(CollectorPP)); } void ModuleDepCollector::attachToASTReader(ASTReader &R) {} diff --git a/clang/test/ClangScanDeps/link-libraries.c b/clang/test/ClangScanDeps/link-libraries.c index cc2e223102024..3719d713e775c 100644 --- a/clang/test/ClangScanDeps/link-libraries.c +++ b/clang/test/ClangScanDeps/link-libraries.c @@ -32,7 +32,7 @@ module transitive { }] // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=root > %t/result.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s // CHECK: { diff --git a/clang/test/ClangScanDeps/modules-full-by-mod-name.c b/clang/test/ClangScanDeps/modules-full-by-mod-name.c index c838614d0bfde..edb99636aaf25 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mod-name.c +++ b/clang/test/ClangScanDeps/modules-full-by-mod-name.c @@ -25,7 +25,7 @@ module transitive { header "transitive.h" } }] // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=root > %t/result.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s // CHECK: { diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c new file mode 100644 index 0000000000000..030f7f3427810 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c @@ -0,0 +1,108 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t + +//--- module.modulemap +module root { header "root.h" } +module direct { header "direct.h" } +module transitive { header "transitive.h" } +module root1 { header "root1.h"} +//--- root.h +#include "direct.h" +#include "root/textual.h" + +//--- root1.h +#include "direct.h" + +//--- direct.h +#include "transitive.h" +//--- transitive.h +// empty + +//--- root/textual.h +// This is here to verify that the "root" directory doesn't clash with name of +// the "root" module. + +//--- cdb.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root,root1,direct > %t/result.json +// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "transitive" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/direct.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "direct" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "direct" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/root.h", +// CHECK-NEXT: "[[PREFIX]]/root/textual.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "root" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "direct" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/root1.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "root1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/transitive.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "transitive" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [] +// CHECK-NEXT: } diff --git a/clang/test/Modules/transitive-system.test b/clang/test/Modules/transitive-system.test index b1f1558b31742..5f6196cc1d6a3 100644 --- a/clang/test/Modules/transitive-system.test +++ b/clang/test/Modules/transitive-system.test @@ -2,9 +2,9 @@ // RUN: split-file %s %t // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=direct > %t/result1.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=direct > %t/result1.json // RUN: rm -rf %t/cache -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=transitive > %t/result2.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=transitive > %t/result2.json // RUN: %deps-to-rsp %t/result1.json --module-name transitive > %t/1.rsp // RUN: %deps-to-rsp %t/result2.json --module-name transitive > %t/2.rsp // RUN: diff %t/1.rsp %t/2.rsp diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index c11a34870b204..5f5bf42df5e6b 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -87,7 +87,7 @@ static std::string ModuleFilesDir; static bool EagerLoadModules; static unsigned NumThreads = 0; static std::string CompilationDB; -static std::optional ModuleName; +static std::optional ModuleNames; static std::vector ModuleDepTargets; static std::string TranslationUnitFile; static bool DeprecatedDriverCommand; @@ -205,8 +205,8 @@ static void ParseArgs(int argc, char **argv) { if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ)) CompilationDB = A->getValue(); - if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_name_EQ)) - ModuleName = A->getValue(); + if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_names_EQ)) + ModuleNames = A->getValue(); for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ)) ModuleDepTargets.emplace_back(A->getValue()); @@ -664,6 +664,16 @@ static bool handleModuleResult(StringRef ModuleName, return false; } +static void handleErrorWithInfoString(StringRef Info, llvm::Error E, + SharedStream &OS, SharedStream &Errs) { + llvm::handleAllErrors(std::move(E), [&Info, &Errs](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error: " << Info << ":\n"; + OS << Err.getMessage(); + }); + }); +} + class P1689Deps { public: void printDependencies(raw_ostream &OS) { @@ -1008,7 +1018,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { }; if (Format == ScanningOutputFormat::Full) - FD.emplace(!ModuleName ? Inputs.size() : 0); + FD.emplace(!ModuleNames ? Inputs.size() : 0); std::atomic NumStatusCalls = 0; std::atomic NumOpenFileForReadCalls = 0; @@ -1082,13 +1092,48 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { MakeformatOS, Errs)) HadErrors = true; } - } else if (ModuleName) { - auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( - *ModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput); - if (handleModuleResult(*ModuleName, MaybeModuleDepsGraph, *FD, - LocalIndex, DependencyOS, Errs)) - HadErrors = true; + } else if (ModuleNames) { + StringRef ModuleNameRef(*ModuleNames); + SmallVector Names; + ModuleNameRef.split(Names, ','); + + if (Names.size() == 1) { + auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( + Names[0], Input->CommandLine, CWD, AlreadySeenModules, + LookupOutput); + if (handleModuleResult(Names[0], MaybeModuleDepsGraph, *FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } else { + if (llvm::Error Err = + WorkerTool.initializeCompilerInstanceWithContext( + CWD, Input->CommandLine)) { + handleErrorWithInfoString( + "Compiler instance with context setup error", std::move(Err), + DependencyOS, Errs); + HadErrors = true; + continue; + } + + for (auto N : Names) { + auto MaybeModuleDepsGraph = + WorkerTool.computeDependenciesByNameWithContext( + N, AlreadySeenModules, LookupOutput); + if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex, + DependencyOS, Errs)) { + HadErrors = true; + break; + } + } + + if (llvm::Error Err = + WorkerTool.finalizeCompilerInstanceWithContext()) { + handleErrorWithInfoString( + "Compiler instance with context finialization error", + std::move(Err), DependencyOS, Errs); + HadErrors = true; + } + } } else { std::unique_ptr TU; std::optional TUBuffer; diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index 7a63b18f6d462..6ea9d824c9646 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -26,7 +26,9 @@ def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of laz def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">; defm compilation_database : Eq<"compilation-database", "Compilation database">; -defm module_name : Eq<"module-name", "the module of which the dependencies are to be computed">; +defm module_names + : Eq<"module-names", "A comma separated list of names of modules of which " + "the dependencies are to be computed">; defm dependency_target : Eq<"dependency-target", "The names of dependency targets for the dependency file">; defm tu_buffer_path: Eq<"tu-buffer-path", "The path to the translation unit for depscan. Not compatible with -module-name">;