From e56ac04dde57af76328e4e283e01a6b642267eb0 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 21 Jul 2025 11:16:33 -0700 Subject: [PATCH 1/8] [IncludeTree] Do not pass compilation directories When using include-tree, all the relative paths references should be turned into absolute path already. There is no need to pass options like `-fdebug-compilation-dir` for debug info generation as it makes cache hit less likely when two identical compilations where performed with different compilation directories. --- .../lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp | 3 +++ clang/test/ClangScanDeps/include-tree-preserve-pch-path.c | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 9e3377381a489..0612430ea2fde 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -103,6 +103,9 @@ void tooling::dependencies::configureInvocationForCaching( // Clear APINotes options. CI.getAPINotesOpts().ModuleSearchPaths = {}; + // Reset debug/coverage compilation dir as include-tree uses absolute path. + CodeGenOpts.DebugCompilationDir.clear(); + CodeGenOpts.CoverageCompilationDir.clear(); // Update output paths, and clear working directory. auto CWD = FileSystemOpts.WorkingDir; updateRelativePath(FrontendOpts.OutputFile, CWD); diff --git a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c index c9331dea2dc51..65e3f156cdaeb 100644 --- a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c +++ b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c @@ -1,5 +1,6 @@ // REQUIRES: ondisk_cas +// RUN: rm -rf %t // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json @@ -10,9 +11,10 @@ // CHECK: "-fmodule-format=obj" // CHECK: "-dwarf-ext-refs" +// CHECK-NOT: -fdebug-compilation-dir // RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp -// RUN: %clang @%t/pch.rsp +// RUN: %clang @%t/pch.rsp -Rcompile-job-cache // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_tu.json // RUN: FileCheck %s -input-file %t/deps_tu.json -DPREFIX=%/t @@ -21,7 +23,9 @@ // RUN: %clang @%t/tu.rsp // RUN: cat %t/tu.ll | FileCheck %s -check-prefix=LLVMIR -DPREFIX=%/t -// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "prefix.pch" +// LLVMIR: !DIFile(filename: "[[PREFIX]]/tu.c", directory: "") +// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "[[PREFIX]]/prefix.pch" +// LLVMIR: !DIFile(filename: "prefix.h", directory: "") // Extract include-tree casid // RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid From 8ca8f5274e43e5d6d566f1cda2341e9009f8a83f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 24 Jul 2025 11:23:04 -0700 Subject: [PATCH 2/8] Use CASID to encode splitDwarfFilename --- .../include/clang/Basic/ASTSourceDescriptor.h | 6 +- .../include/clang/Basic/DiagnosticCASKinds.td | 6 +- clang/include/clang/Basic/Module.h | 10 + .../include/clang/Frontend/CompilerInstance.h | 4 +- clang/include/clang/Serialization/ASTReader.h | 10 + .../clang/Serialization/InMemoryModuleCache.h | 11 +- .../include/clang/Serialization/ModuleFile.h | 3 + clang/lib/Basic/ASTSourceDescriptor.cpp | 1 + clang/lib/CodeGen/CGDebugInfo.cpp | 5 + .../CodeGen/ObjectFilePCHContainerWriter.cpp | 4 +- clang/lib/Frontend/CompilerInstance.cpp | 102 +++++++-- clang/lib/Frontend/FrontendAction.cpp | 18 +- clang/lib/Serialization/ASTReader.cpp | 42 +++- clang/lib/Serialization/ASTWriter.cpp | 8 +- .../lib/Serialization/InMemoryModuleCache.cpp | 13 +- .../DependencyScanningWorker.cpp | 10 +- .../IncludeTreeActionController.cpp | 7 +- .../DependencyScanning/ModuleDepCollector.cpp | 18 +- .../DependencyScanning/ScanAndUpdateArgs.cpp | 12 +- clang/test/CAS/depscan-include-tree.c | 4 +- .../ClangScanDeps/cas-fs-multiple-commands.c | 12 +- clang/test/ClangScanDeps/gmodules.c | 215 ++++++++++++++++++ .../include-tree-multiple-commands.c | 12 +- .../include-tree-preserve-pch-path.c | 21 +- .../ClangScanDeps/modules-cas-context-hash.c | 6 +- .../modules-cas-fs-prefix-mapping.c | 8 +- .../modules-cas-trees-with-pch.c | 10 +- clang/test/ClangScanDeps/modules-cas-trees.c | 8 +- .../modules-include-tree-dependency-file.c | 2 +- ...include-tree-pch-common-stale-prefix-map.c | 192 ++++++++++++++++ .../modules-include-tree-prefix-map.c | 16 +- .../modules-include-tree-with-pch.c | 8 +- .../test/ClangScanDeps/modules-include-tree.c | 16 +- .../modules-pch-cas-fs-prefix-mapping.c | 16 +- llvm/tools/dsymutil/BinaryHolder.cpp | 44 +++- llvm/tools/dsymutil/BinaryHolder.h | 9 +- llvm/tools/dsymutil/Options.td | 5 + llvm/tools/dsymutil/dsymutil.cpp | 21 +- 38 files changed, 762 insertions(+), 153 deletions(-) create mode 100644 clang/test/ClangScanDeps/gmodules.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c diff --git a/clang/include/clang/Basic/ASTSourceDescriptor.h b/clang/include/clang/Basic/ASTSourceDescriptor.h index 175e0551db765..c0ce5486d4d4a 100644 --- a/clang/include/clang/Basic/ASTSourceDescriptor.h +++ b/clang/include/clang/Basic/ASTSourceDescriptor.h @@ -30,14 +30,15 @@ class ASTSourceDescriptor { StringRef Path; StringRef ASTFile; ASTFileSignature Signature; + StringRef CASID; Module *ClangModule = nullptr; public: ASTSourceDescriptor() = default; ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile, - ASTFileSignature Signature) + ASTFileSignature Signature, StringRef CASID) : PCHModuleName(std::move(Name)), Path(std::move(Path)), - ASTFile(std::move(ASTFile)), Signature(Signature) {} + ASTFile(std::move(ASTFile)), Signature(Signature), CASID(CASID) {} ASTSourceDescriptor(Module &M); std::string getModuleName() const; @@ -45,6 +46,7 @@ class ASTSourceDescriptor { StringRef getASTFile() const { return ASTFile; } ASTFileSignature getSignature() const { return Signature; } Module *getModuleOrNull() const { return ClangModule; } + StringRef getCASID() const { return CASID; } }; } // namespace clang diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index 7dd533c17d53d..f37b134859ff7 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -23,8 +23,10 @@ def err_cas_depscan_daemon_connection: Error< def err_cas_depscan_failed: Error< "CAS-based dependency scan failed: %0">, DefaultFatal; def err_cas_store: Error<"failed to store to CAS: %0">, DefaultFatal; -def err_cas_unloadable_module : Error< - "module file '%0' not found: unloadable module cache key %1: %2">, DefaultFatal; +def err_cas_unloadable_module + : Error<"module file '%0' not found: unloadable %select{casid|module cache " + "key}1 %2: %3">, + DefaultFatal; def err_cas_missing_module : Error< "module file '%0' not found: missing module cache key %1: %2">, DefaultFatal; def err_cas_missing_root_id : Error< diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index cddf9d2b81ce8..eacab0e106924 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -274,6 +274,9 @@ class alignas(8) Module { /// The \c ActionCache key for this module, if any. std::optional ModuleCacheKey; + /// The \c CASID for the loaded module, otherwise empty. + std::string CASID; + /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -773,6 +776,13 @@ class alignas(8) Module { getTopLevelModule()->ModuleCacheKey = std::move(Key); } + StringRef getCASID() const { return getTopLevelModule()->CASID; } + + void setCASID(std::string ID) { + assert(getCASID().empty() || getCASID() == ID); + getTopLevelModule()->CASID = std::move(ID); + } + /// Retrieve the umbrella directory as written. std::optional getUmbrellaDirAsWritten() const { if (const auto *Dir = std::get_if(&Umbrella)) diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index e59855f22d8df..a72feb1edab9d 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -1001,8 +1001,8 @@ class CompilerInstance : public ModuleLoader { /// "-fmodule-file-cache-key", or an imported pcm file. Used in diagnostics. /// /// \returns true on failure. - bool addCachedModuleFile(StringRef Path, StringRef CacheKey, - StringRef Provider); + bool addCachedModuleFile(StringRef Path, StringRef CASID, StringRef Provider, + bool IsKey); ModuleCache &getModuleCache() const { return *ModCache; } diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index a38572531533b..2843dd1f4f740 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -286,6 +286,14 @@ class ASTReaderListener { return false; } + /// Called for each module CASID. + /// + /// \returns true to indicate the key cannot be loaded. + virtual bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) { + return false; + } + /// Indicates that a particular module file extension has been read. virtual void readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) {} @@ -342,6 +350,8 @@ class ChainedASTReaderListener : public ASTReaderListener { bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override; void readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) override; }; diff --git a/clang/include/clang/Serialization/InMemoryModuleCache.h b/clang/include/clang/Serialization/InMemoryModuleCache.h index fc3ba334fc64d..8aa3a48db6542 100644 --- a/clang/include/clang/Serialization/InMemoryModuleCache.h +++ b/clang/include/clang/Serialization/InMemoryModuleCache.h @@ -30,6 +30,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase { struct PCM { std::unique_ptr Buffer; + std::string CASID; + /// Track whether this PCM is known to be good (either built or /// successfully imported by a CompilerInstance/ASTReader using this /// cache). @@ -38,6 +40,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase { PCM() = default; PCM(std::unique_ptr Buffer) : Buffer(std::move(Buffer)) {} + + PCM(std::unique_ptr Buffer, llvm::StringRef CASID) + : Buffer(std::move(Buffer)), CASID(CASID.str()) {} }; /// Cache of buffers. @@ -64,7 +69,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase { /// \post state is Tentative /// \return a reference to the buffer as a convenience. llvm::MemoryBuffer &addPCM(llvm::StringRef Filename, - std::unique_ptr Buffer); + std::unique_ptr Buffer, + llvm::StringRef CASID = ""); /// Store a just-built PCM under the Filename. /// @@ -90,6 +96,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase { /// Get a pointer to the pCM if it exists; else nullptr. llvm::MemoryBuffer *lookupPCM(llvm::StringRef Filename) const; + /// Get the PCM if it exits; else nullptr. + const PCM *lookup(llvm::StringRef Filename) const; + /// Check whether the PCM is final and has been shown to work. /// /// \return true iff state is Final. diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 4eeed6a65dad1..c9dd1c8bb1985 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -147,6 +147,9 @@ class ModuleFile { /// The \c ActionCache key for this module, or empty. std::string ModuleCacheKey; + /// The \c CASID for the module, or empty. + std::string CASID; + /// The CAS filesystem root ID for implicit modules built with the dependency /// scanner, or empty. std::string CASFileSystemRootID; diff --git a/clang/lib/Basic/ASTSourceDescriptor.cpp b/clang/lib/Basic/ASTSourceDescriptor.cpp index 8072c08a51d3a..6cecc8347d911 100644 --- a/clang/lib/Basic/ASTSourceDescriptor.cpp +++ b/clang/lib/Basic/ASTSourceDescriptor.cpp @@ -21,6 +21,7 @@ ASTSourceDescriptor::ASTSourceDescriptor(Module &M) Path = M.Directory->getName(); if (auto File = M.getASTFile()) ASTFile = File->getName(); + CASID = M.getCASID(); } std::string ASTSourceDescriptor::getModuleName() const { diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 65e6d8c24acaa..525e9f161dcdd 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3348,6 +3348,11 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod, PCM = Mod.getPath(); } llvm::sys::path::append(PCM, Mod.getASTFile()); + + // FIXME: Prefer CASID if exists. + if (!Mod.getCASID().empty()) + PCM = Mod.getCASID(); + DIB.createCompileUnit( TheCU->getSourceLanguage(), // TODO: Support "Source" from external AST providers? diff --git a/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp b/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp index fafcae1f1e636..49f19061cb0ea 100644 --- a/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp +++ b/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp @@ -191,8 +191,8 @@ class PCHContainerGenerator : public ASTConsumer { // Prepare CGDebugInfo to emit debug info for a clang module. auto *DI = Builder->getModuleDebugInfo(); StringRef ModuleName = llvm::sys::path::filename(MainFileName); - DI->setPCHDescriptor( - {ModuleName, "", OutputFileName, ASTFileSignature::createDISentinel()}); + DI->setPCHDescriptor({ModuleName, "", OutputFileName, + ASTFileSignature::createDISentinel(), /*CASID=*/""}); DI->setModuleMap(MMap); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 76150504bd553..ac38f097f2c41 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -646,6 +646,8 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override; private: bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID, @@ -2600,25 +2602,51 @@ void CompilerInstance::setExternalSemaSource( ExternalSemaSrc = std::move(ESS); } -static bool addCachedModuleFileToInMemoryCache( - StringRef Path, StringRef CacheKey, StringRef Provider, - cas::ObjectStore &CAS, cas::ActionCache &Cache, - ModuleCache &ModCache, DiagnosticsEngine &Diags) { +static bool addCachedModuleFileToInMemoryCache(StringRef Path, + cas::ObjectStore &CAS, + cas::ObjectRef Object, + ModuleCache &ModCache, + DiagnosticsEngine &Diags) { + // FIXME: We wait to materialize each module file before proceeding, which + // introduces latency for a network CAS. Instead we should collect all the + // module keys and materialize them concurrently using \c getProxyFuture, for + // better network utilization. + auto OutputProxy = CAS.getProxy(Object); + if (!OutputProxy) { + Diags.Report(diag::err_cas_unloadable_module) + << Path << 0 << CAS.getID(Object).toString() << OutputProxy.takeError(); + return true; + } + + auto PCMID = OutputProxy->getID().toString(); + if (const auto *PCM = ModCache.getInMemoryModuleCache().lookup(Path)) { + // If the CASID in the module cache differs, return error. + if (!PCM->CASID.empty() && PCM->CASID != PCMID) + return true; - if (ModCache.getInMemoryModuleCache().lookupPCM(Path)) return false; + } + + ModCache.getInMemoryModuleCache().addPCM(Path, OutputProxy->getMemoryBuffer(), + OutputProxy->getID().toString()); + return false; +} +static bool addCachedModuleFileToInMemoryCacheFromKey( + StringRef Path, StringRef CacheKey, StringRef Provider, + cas::ObjectStore &CAS, cas::ActionCache &Cache, ModuleCache &ModCache, + DiagnosticsEngine &Diags) { auto ID = CAS.parseID(CacheKey); if (!ID) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << ID.takeError(); + << Path << 1 << CacheKey << ID.takeError(); return true; } auto Value = Cache.get(*ID); if (!Value) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << Value.takeError(); + << Path << 1 << CacheKey << Value.takeError(); return true; } if (!*Value) { @@ -2639,7 +2667,7 @@ static bool addCachedModuleFileToInMemoryCache( auto ValueRef = CAS.getReference(**Value); if (!ValueRef) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << "result module cannot be loaded from CAS"; + << Path << 1 << CacheKey << "result module cannot be loaded from CAS"; return true; } @@ -2648,44 +2676,68 @@ static bool addCachedModuleFileToInMemoryCache( cas::CompileJobResultSchema Schema(CAS); if (llvm::Error E = Schema.load(*ValueRef).moveInto(Result)) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << std::move(E); + << Path << 1 << CacheKey << std::move(E); return true; } auto Output = Result->getOutput(cas::CompileJobCacheResult::OutputKind::MainOutput); if (!Output) llvm::report_fatal_error("missing main output"); - // FIXME: We wait to materialize each module file before proceeding, which - // introduces latency for a network CAS. Instead we should collect all the - // module keys and materialize them concurrently using \c getProxyFuture, for - // better network utilization. - auto OutputProxy = CAS.getProxy(Output->Object); - if (!OutputProxy) { + + return addCachedModuleFileToInMemoryCache(Path, CAS, Output->Object, ModCache, + Diags); +} + +static bool addCachedModuleFileToInMemoryCacheFromID(StringRef Filename, + cas::ObjectStore &CAS, + StringRef CASID, + ModuleCache &ModCache, + DiagnosticsEngine &Diags) { + auto ID = CAS.parseID(CASID); + if (!ID) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << OutputProxy.takeError(); + << Filename << 0 << CASID << ID.takeError(); return true; } + auto ModuleRef = CAS.getReference(*ID); + if (!ModuleRef) { + Diags.Report(diag::err_cas_unloadable_module) + << Filename << 1 << CASID << "does not exist in CAS"; - ModCache.getInMemoryModuleCache().addPCM(Path, - OutputProxy->getMemoryBuffer()); - return false; + return true; + } + + return addCachedModuleFileToInMemoryCache(Filename, CAS, *ModuleRef, ModCache, + Diags); } -bool CompilerInstance::addCachedModuleFile(StringRef Path, StringRef CacheKey, - StringRef Provider) { - return addCachedModuleFileToInMemoryCache( - Path, CacheKey, Provider, getOrCreateObjectStore(), - getOrCreateActionCache(), getModuleCache(), getDiagnostics()); +bool CompilerInstance::addCachedModuleFile(StringRef Path, StringRef CASID, + StringRef Provider, bool IsKey) { + if (IsKey) + return addCachedModuleFileToInMemoryCacheFromKey( + Path, CASID, Provider, getOrCreateObjectStore(), + getOrCreateActionCache(), getModuleCache(), getDiagnostics()); + + return addCachedModuleFileToInMemoryCacheFromID( + Path, getOrCreateObjectStore(), CASID, getModuleCache(), + getDiagnostics()); } bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) { // FIXME: add name/path of the importing module? - return addCachedModuleFileToInMemoryCache( + return addCachedModuleFileToInMemoryCacheFromKey( Filename, CacheKey, "imported module", CAS, Cache, ModCache, Diags); } +bool CompileCacheASTReaderHelper::readModuleCASID(StringRef ModuleName, + StringRef Filename, + StringRef CASID) { + return addCachedModuleFileToInMemoryCacheFromID(Filename, CAS, CASID, + ModCache, Diags); +} + /// Verify that ID is in the CAS. Otherwise the module cache probably was /// created with a different CAS. bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID, diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 821472eaf89c7..782efced9bef5 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -1345,7 +1345,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // Provide any modules from the action cache. for (const auto &KeyPair : CI.getFrontendOpts().ModuleCacheKeys) if (CI.addCachedModuleFile(KeyPair.first, KeyPair.second, - "-fmodule-file-cache-key")) + "-fmodule-file-cache-key", + /*IsKey=*/true)) return false; @@ -1410,9 +1411,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, DeserialListener, DeleteDeserialListener); DeleteDeserialListener = true; } + std::string PCHCASID; if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty() || IncludeTreePCH) { - StringRef PCHPath; + std::string PCHPath; DisableValidationForModuleKind DisableValidation; std::unique_ptr PCHBuffer; if (IncludeTreePCH) { @@ -1421,7 +1423,11 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, if (llvm::Error E = IncludeTreePCH->getMemoryBuffer().moveInto(PCHBuffer)) return reportError(std::move(E)); - PCHPath = PCHBuffer->getBufferIdentifier(); + auto PCHFile = IncludeTreePCH->getContents(); + if (!PCHFile) + return reportError(PCHFile.takeError()); + PCHCASID = PCHFile->getID().toString(); + PCHPath = PCHCASID; } else { PCHPath = CI.getPreprocessorOpts().ImplicitPCHInclude; DisableValidation = @@ -1433,6 +1439,12 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, DeserialListener, DeleteDeserialListener, std::move(PCHBuffer)); if (!CI.getASTContext().getExternalSource()) return false; + + // After loading PCH, set its CASID for content. + if (!PCHCASID.empty()) { + auto &MF = CI.getASTReader()->getModuleManager().getPrimaryModule(); + MF.CASID = PCHCASID; + } } // If modules are enabled, create the AST reader before creating // any builtins, so that all declarations know that they might be diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index a22ccdfc3bc53..0b1978590d78d 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -276,6 +276,12 @@ bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName, return First->readModuleCacheKey(ModuleName, Filename, CacheKey) || Second->readModuleCacheKey(ModuleName, Filename, CacheKey); } +bool ChainedASTReaderListener::readModuleCASID(StringRef ModuleName, + StringRef Filename, + StringRef CASID) { + return First->readModuleCASID(ModuleName, Filename, CASID) || + Second->readModuleCASID(ModuleName, Filename, CASID); +} void ChainedASTReaderListener::readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) { @@ -3382,7 +3388,7 @@ ASTReader::ReadControlBlock(ModuleFile &F, std::string ImportedFile; std::string StoredFile; bool IgnoreImportedByNote = false; - StringRef ImportedCacheKey; + StringRef ImportedCASKey; // For prebuilt and explicit modules first consult the file map for // an override. Note that here we don't search prebuilt module @@ -3430,18 +3436,30 @@ ASTReader::ReadControlBlock(ModuleFile &F, } } - ImportedCacheKey = ReadStringBlob(Record, Idx, Blob); + ImportedCASKey = ReadStringBlob(Record, Idx, Blob); } - if (!ImportedCacheKey.empty()) { - if (!Listener || Listener->readModuleCacheKey( - ImportedName, ImportedFile, ImportedCacheKey)) { + if (!ImportedCASKey.empty()) { + if (!Listener) { Diag(diag::err_ast_file_not_found) << moduleKindForDiagnostic(ImportedKind) << ImportedFile << true - << ("missing or unloadable module cache key" + ImportedCacheKey) + << ("missing listener to read CAS reference" + ImportedCASKey) .str(); return Failure; } + if (ImportedKind == MK_ImplicitModule) { + if (Listener->readModuleCacheKey(ImportedName, ImportedFile, + ImportedCASKey)) { + // If the module cache key failed to read, treat the module as + // out-of-date so it can be rebuilt. + return OutOfDate; + } + } else { + if (Listener->readModuleCASID(ImportedName, ImportedFile, + ImportedCASKey)) { + return OutOfDate; + } + } } // If our client can't cope with us being out of date, we can't cope with @@ -6150,7 +6168,7 @@ bool ASTReader::readASTFileControlBlock( auto Filename = ResolveImportedPath(PathBuf, FilenameStr, ModuleDir); StringRef CacheKey = ReadStringBlob(Record, Idx, Blob); if (!CacheKey.empty()) - Listener.readModuleCacheKey(ModuleName, *Filename, CacheKey); + Listener.readModuleCASID(ModuleName, *Filename, CacheKey); Listener.visitImport(ModuleName, *Filename); break; } @@ -6373,6 +6391,14 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; if (!F.ModuleCacheKey.empty()) CurrentModule->setModuleCacheKey(F.ModuleCacheKey); + + // Set CASID for current module. + if (const auto *PCM = + ModuleMgr.getModuleCache().getInMemoryModuleCache().lookup( + F.FileName)) { + CurrentModule->setCASID(PCM->CASID); + F.CASID = PCM->CASID; + } } CurrentModule->Kind = Kind; @@ -9984,7 +10010,7 @@ std::optional ASTReader::getSourceDescriptor(unsigned ID) { StringRef FileName = llvm::sys::path::filename(MF.FileName); return ASTSourceDescriptor(ModuleName, llvm::sys::path::parent_path(MF.FileName), - FileName, MF.Signature); + FileName, MF.Signature, MF.CASID); } return std::nullopt; } diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 52193c8ca7d5e..d0e1c0bc65e34 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1660,7 +1660,13 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, StringRef isysroot) { llvm::append_range(Blob, M.Signature); AddPathBlob(M.FileName, Record, Blob); - AddStringBlob(M.ModuleCacheKey, Record, Blob); + if (M.Kind == serialization::MK_ImplicitModule) + AddStringBlob(M.ModuleCacheKey, Record, Blob); + else { + assert(!(M.CASID.empty() && !M.ModuleCacheKey.empty()) && + "should not have module cache key without CASID"); + AddStringBlob(M.CASID, Record, Blob); + } } Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob); diff --git a/clang/lib/Serialization/InMemoryModuleCache.cpp b/clang/lib/Serialization/InMemoryModuleCache.cpp index d35fa2a807f4d..556f9453a1351 100644 --- a/clang/lib/Serialization/InMemoryModuleCache.cpp +++ b/clang/lib/Serialization/InMemoryModuleCache.cpp @@ -23,8 +23,9 @@ InMemoryModuleCache::getPCMState(llvm::StringRef Filename) const { llvm::MemoryBuffer & InMemoryModuleCache::addPCM(llvm::StringRef Filename, - std::unique_ptr Buffer) { - auto Insertion = PCMs.insert(std::make_pair(Filename, std::move(Buffer))); + std::unique_ptr Buffer, + llvm::StringRef CASID) { + auto Insertion = PCMs.try_emplace(Filename, PCM{std::move(Buffer), CASID}); assert(Insertion.second && "Already has a PCM"); return *Insertion.first->second.Buffer; } @@ -48,6 +49,14 @@ InMemoryModuleCache::lookupPCM(llvm::StringRef Filename) const { return I->second.Buffer.get(); } +const InMemoryModuleCache::PCM * +InMemoryModuleCache::lookup(llvm::StringRef Filename) const { + auto I = PCMs.find(Filename); + if (I == PCMs.end()) + return nullptr; + return &I->second; +} + bool InMemoryModuleCache::isPCMFinal(llvm::StringRef Filename) const { return getPCMState(Filename) == Final; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index e66106d4f0b13..2b60468156ce6 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -207,7 +207,15 @@ class PrebuiltModuleListener : public ASTReaderListener { CI.getFrontendOpts().ModuleCacheKeys.emplace_back(std::string(Filename), std::string(CacheKey)); // FIXME: add name/path of the importing module? - return CI.addCachedModuleFile(Filename, CacheKey, "imported module"); + return CI.addCachedModuleFile(Filename, CacheKey, "imported module", + /*IsKey=*/true); + } + + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override { + // FIXME: add name/path of the importing module? + return CI.addCachedModuleFile(Filename, CASID, "imported module", + /*IsKey=*/false); } private: diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index e27bd3e218017..e1a5f15b60bff 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -705,12 +705,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, if (!CASContents) return llvm::errorCodeToError(CASContents.getError()); - StringRef PCHFilename = ""; - if (NewInvocation.getFrontendOpts().IncludeTreePreservePCHPath) - PCHFilename = PPOpts.ImplicitPCHInclude; - - auto PCHFile = - cas::IncludeTree::File::create(DB, PCHFilename, **CASContents); + auto PCHFile = cas::IncludeTree::File::create(DB, "", **CASContents); if (!PCHFile) return PCHFile.takeError(); PCHRef = PCHFile->getRef(); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index e0301b865c470..3e798000c874d 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -363,10 +363,14 @@ ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs( // Report the prebuilt modules this module uses. for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps) { - CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile); - if (PrebuiltModule.ModuleCacheKey) + if (PrebuiltModule.ModuleCacheKey) { + // canonicalize the PCM path if using CAS. + auto PCMFile = llvm::sys::path::filename(PrebuiltModule.PCMFile); + CI.getMutFrontendOpts().ModuleFiles.push_back(PCMFile.str()); CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back( - PrebuiltModule.PCMFile, *PrebuiltModule.ModuleCacheKey); + PCMFile, *PrebuiltModule.ModuleCacheKey); + } else + CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile); } // Add module file inputs from dependencies. @@ -421,9 +425,11 @@ void ModuleDepCollector::addModuleFiles( Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile); assert(MD && "Inconsistent dependency info"); - if (MD->ModuleCacheKey) + if (MD->ModuleCacheKey) { + PCMPath = llvm::sys::path::filename(PCMPath); CI.getFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); + } if (Service.shouldEagerLoadModules()) CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else @@ -440,9 +446,11 @@ void ModuleDepCollector::addModuleFiles( Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile); assert(MD && "Inconsistent dependency info"); - if (MD->ModuleCacheKey) + if (MD->ModuleCacheKey) { + PCMPath = llvm::sys::path::filename(PCMPath); CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); + } if (Service.shouldEagerLoadModules()) CI.getMutFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 0612430ea2fde..12f9d46c308cd 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -93,13 +93,6 @@ void tooling::dependencies::configureInvocationForCaching( PPOpts.MacroIncludes.clear(); PPOpts.Includes.clear(); } - if (!FrontendOpts.IncludeTreePreservePCHPath) { - // Disable `-gmodules` to avoid debug info referencing a non-existent PCH - // filename. - // FIXME: we should also allow -gmodules if there is no PCH involved. - CodeGenOpts.DebugTypeExtRefs = false; - HSOpts.ModuleFormat = "raw"; - } // Clear APINotes options. CI.getAPINotesOpts().ModuleSearchPaths = {}; @@ -113,6 +106,9 @@ void tooling::dependencies::configureInvocationForCaching( updateRelativePath(CI.getDiagnosticOpts().DiagnosticLogFile, CWD); updateRelativePath(CI.getDependencyOutputOpts().OutputFile, CWD); FileSystemOpts.WorkingDir.clear(); + + // IncludeTree always use absolute path. Do not set debug comp dir. + CodeGenOpts.DebugCompilationDir.clear(); break; } case CachingInputKind::FileSystemRoot: { @@ -213,8 +209,6 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, mapInPlaceAll(FrontendOpts.ASTMergeFiles); Mapper.mapInPlace(FrontendOpts.OverrideRecordLayoutsFile); Mapper.mapInPlace(FrontendOpts.StatsFile); - for (auto &[Path, _] : FrontendOpts.ModuleCacheKeys) - Mapper.mapInPlace(Path); // Filesystem options. Mapper.mapInPlace(FileSystemOpts.WorkingDir); diff --git a/clang/test/CAS/depscan-include-tree.c b/clang/test/CAS/depscan-include-tree.c index a75fb2e0f8b72..628993b6ca0a2 100644 --- a/clang/test/CAS/depscan-include-tree.c +++ b/clang/test/CAS/depscan-include-tree.c @@ -14,6 +14,8 @@ // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK: "-fcas-include-tree" // CHECK: "-isysroot" +// CHECK: "-fmodule-format=obj" +// CHECK: "-dwarf-ext-refs" // CHECK: "-coverage-data-file=[[PREFIX]]/t.gcda" // CHECK: "-coverage-notes-file=[[PREFIX]]/t.gcno" // SHOULD-NOT: "-fcas-fs" @@ -21,8 +23,6 @@ // SHOULD-NOT: "-I" // SHOULD-NOT: "[[PREFIX]]/t.c" // SHOULD-NOT: "-D" -// SHOULD-NOT: "-dwarf-ext-refs" -// SHOULD-NOT: "-fmodule-format=obj" // RUN: FileCheck %s -input-file %t/inline.d -check-prefix=DEPS -DPREFIX=%t diff --git a/clang/test/ClangScanDeps/cas-fs-multiple-commands.c b/clang/test/ClangScanDeps/cas-fs-multiple-commands.c index 28bf3857cf334..09e0ad5a330f4 100644 --- a/clang/test/ClangScanDeps/cas-fs-multiple-commands.c +++ b/clang/test/ClangScanDeps/cas-fs-multiple-commands.c @@ -109,12 +109,12 @@ // CHECK-NOT: "-fcas-input-file-cache-key" // CHECK: "-E" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c" // CHECK: "[[PREFIX]]/tu.c" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "file-deps": [ // CHECK-NEXT: "[[PREFIX]]/tu.c" @@ -139,12 +139,12 @@ // CHECK-NEXT: "[[CPP_CACHE_KEY]]" // CHECK: "-emit-llvm-bc" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c-cpp-output" // CHECK-NOT: "{{.*}}tu.i" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "input-file": "[[PREFIX]]{{.}}tu.c" // CHECK-NEXT: } @@ -217,7 +217,7 @@ // CHECK-LIBCLANG-NEXT: [[PREFIX]]/tu.c // CHECK-LIBCLANG-NOT: -fcas-input-file-cache-key // CHECK-LIBCLANG-NOT: {{.*}}tu.c -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 1: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. @@ -229,7 +229,7 @@ // CHECK-LIBCLANG-NEXT: [[PREFIX]]/tu.c // CHECK-LIBCLANG-NOT: -fcas-fs // CHECK-LIBCLANG-NOT: {{.*}}tu.i -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c-cpp-output {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c-cpp-output {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 2: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c new file mode 100644 index 0000000000000..6f5c827e4f246 --- /dev/null +++ b/clang/test/ClangScanDeps/gmodules.c @@ -0,0 +1,215 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +/// Scan PCH +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +/// Build PCH +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Top.rsp +// RUN: %clang @%t/Left.rsp +// RUN: %clang @%t/pch.rsp + +/// Scan TU with PCH +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +/// Build TU +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Right.rsp +// RUN: %clang @%t/tu.rsp + +/// Check debug info is correct. +// RUN: %clang %t/tu.o -o %t/a.out +// RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty +// WARN-NOT: warning: + +// RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF + +/// Check module in a different directory and compare output. +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch_2.json +// RUN: %deps-to-rsp %t/deps_pch_2.json --module-name Top > %t/Top_2.rsp +// RUN: %deps-to-rsp %t/deps_pch_2.json --module-name Left > %t/Left_2.rsp +// RUN: %deps-to-rsp %t/deps_pch_2.json --tu-index 0 > %t/pch_2.rsp +// RUN: %clang @%t/Top_2.rsp +// RUN: %clang @%t/Left_2.rsp +// RUN: %clang @%t/pch_2.rsp -o %t/prefix_2.pch +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_2.json +// RUN: %deps-to-rsp %t/deps_2.json --module-name Right > %t/Right_2.rsp +// RUN: %deps-to-rsp %t/deps_2.json --tu-index 0 > %t/tu_2.rsp +// RUN: %clang @%t/Right_2.rsp +// RUN: %clang @%t/tu_2.rsp + +// Check all the types are available +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Top") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Top") +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Left") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Left") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Right") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Right") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Prefix") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" + + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file=Top-{{.*}}.pcm" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NOT: "clang-modulemap-file" +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file= +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fcache-compile-job" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -gmodules -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -c -o DIR/tu.o DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -gmodules -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Left.h +#pragma once +#include "Top.h" +struct Left { struct Top top; }; + +//--- Right.h +#pragma once +#include "Top.h" +struct Right { struct Top top; }; + +//--- prefix.h +#include "Left.h" +struct Prefix { struct Top top; }; + +//--- tu.c +#include "Right.h" + +int main(void) { + struct Left _left; + struct Right _right; + struct Top _top; + struct Prefix _prefix; +} diff --git a/clang/test/ClangScanDeps/include-tree-multiple-commands.c b/clang/test/ClangScanDeps/include-tree-multiple-commands.c index e13a42f8d7727..a3868d948907b 100644 --- a/clang/test/ClangScanDeps/include-tree-multiple-commands.c +++ b/clang/test/ClangScanDeps/include-tree-multiple-commands.c @@ -104,12 +104,12 @@ // CHECK-NOT: "-fcas-input-file-cache-key" // CHECK: "-E" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c" // CHECK-NOT: "{{.*}}tu.c" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "file-deps": [ // CHECK-NEXT: "[[SRC]]/tu.c" @@ -137,12 +137,12 @@ // CHECK-NEXT: "[[CPP_CACHE_KEY]]" // CHECK: "-emit-llvm-bc" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c-cpp-output" // CHECK-NOT: "{{.*}}tu.i" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "input-file": "[[SRC]]/tu.c" // CHECK-NEXT: } @@ -218,7 +218,7 @@ // CHECK-LIBCLANG-NEXT: [[SRC]]/header.h // CHECK-LIBCLANG-NOT: -fcas-input-file-cache-key // CHECK-LIBCLANG-NOT: {{.*}}tu.c -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 1: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. @@ -231,7 +231,7 @@ // CHECK-LIBCLANG-NEXT: [[SRC]]/header.h // CHECK-LIBCLANG-NOT: -fcas-include-tree // CHECK-LIBCLANG-NOT: {{.*}}tu.i -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c-cpp-output {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c-cpp-output {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 2: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. diff --git a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c index 65e3f156cdaeb..e755792c6b1fa 100644 --- a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c +++ b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c @@ -4,7 +4,6 @@ // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: sed -e "s|DIR|%/t|g" %t/cdb_no_preserve.json.template > %t/cdb_no_preserve.json // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_pch.json // RUN: FileCheck %s -input-file %t/deps_pch.json -DPREFIX=%/t @@ -24,38 +23,24 @@ // RUN: cat %t/tu.ll | FileCheck %s -check-prefix=LLVMIR -DPREFIX=%/t // LLVMIR: !DIFile(filename: "[[PREFIX]]/tu.c", directory: "") -// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "[[PREFIX]]/prefix.pch" +// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "llvmcas://{{.*}}" // LLVMIR: !DIFile(filename: "prefix.h", directory: "") // Extract include-tree casid // RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid // RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -check-prefix=INCLUDE_TREE -DPREFIX=%/t -// INCLUDE_TREE: (PCH) [[PREFIX]]/prefix.pch llvmcas:// - -// RUN: clang-scan-deps -compilation-database %t/cdb_no_preserve.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_no_preserve.json -// RUN: FileCheck %s -input-file %t/deps_no_preserve.json -DPREFIX=%/t -check-prefix=NO_PRESERVE - -// Note: "raw" is the default format, so it will not show up in the arguments. -// NO_PRESERVE-NOT: "-fmodule-format= -// NO_PRESERVE-NOT: "-dwarf-ext-refs" +// INCLUDE_TREE: (PCH) llvmcas:// //--- cdb_pch.json.template [{ "directory": "DIR", - "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g -Xclang -finclude-tree-preserve-pch-path", + "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g", "file": "DIR/prefix.h" }] //--- cdb.json.template -[{ - "directory": "DIR", - "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g -Xclang -finclude-tree-preserve-pch-path", - "file": "DIR/tu.c" -}] - -//--- cdb_no_preserve.json.template [{ "directory": "DIR", "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g", diff --git a/clang/test/ClangScanDeps/modules-cas-context-hash.c b/clang/test/ClangScanDeps/modules-cas-context-hash.c index b80709d5acda1..ff2f98746ca5b 100644 --- a/clang/test/ClangScanDeps/modules-cas-context-hash.c +++ b/clang/test/ClangScanDeps/modules-cas-context-hash.c @@ -33,7 +33,7 @@ // CHECK: ], // CHECK: "command-line": [ // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[PREFIX]]/outputs/[[HASH]]/Mod-[[HASH]].pcm" +// CHECK: "Mod-[[HASH]].pcm" // CHECK: "llvmcas://[[KEY]]" // CHECK: ] @@ -57,7 +57,7 @@ // CHECK: ], // CHECK: "command-line": [ // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[PREFIX]]/outputs/[[HASH]]/Mod-[[HASH]].pcm" +// CHECK: "Mod-[[HASH]].pcm" // CHECK: "llvmcas://[[KEY]]" // CHECK: ] @@ -105,4 +105,4 @@ module Mod { header "Mod.h" } //--- Mod.h //--- tu.c -#include "Mod.h" \ No newline at end of file +#include "Mod.h" diff --git a/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c b/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c index 24190438079f4..f9defac1b5d75 100644 --- a/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c +++ b/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c @@ -66,7 +66,7 @@ // CHECK: "-o" // CHECK: "[[PREFIX]]/modules/{{.*}}/A-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}/B-[[B_CONTEXT_HASH:[^.]+]].pcm" +// CHECK: "B-[[B_CONTEXT_HASH:[^.]+]].pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK: "c" @@ -75,7 +75,7 @@ // CHECK: "/^sdk" // CHECK: "-resource-dir" // CHECK: "/^tc/lib/clang/{{.*}}" -// CHECK: "-fmodule-file=B=/^modules/{{.*}}/B-[[B_CONTEXT_HASH]].pcm" +// CHECK: "-fmodule-file=B=B-[[B_CONTEXT_HASH]].pcm" // CHECK: "-isystem" // CHECK: "/^tc/lib/clang/{{.*}}/include" // CHECK: "-internal-externc-isystem" @@ -142,7 +142,7 @@ // CHECK: "/^src" // CHECK: "-fmodule-map-file=/^src/module.modulemap" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}A-{{.*}}.pcm" +// CHECK: "{{.*}}A-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK: "c" @@ -151,7 +151,7 @@ // CHECK: "/^sdk" // CHECK: "-resource-dir" // CHECK: "/^tc/lib/clang/{{.*}}" -// CHECK: "-fmodule-file=A=/^modules/{{.*}}/A-{{.*}}.pcm" +// CHECK: "-fmodule-file=A=A-{{.*}}.pcm" // CHECK: "-isystem" // CHECK: "/^tc/lib/clang/{{.*}}/include" // CHECK: "-internal-externc-isystem" diff --git a/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c b/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c index d936e618ba770..5682293038946 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c +++ b/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c @@ -64,11 +64,11 @@ // PCH: "-fcas-fs" // PCH-NEXT: "[[A_ROOT_ID]]" // PCH: "-o" -// PCH-NEXT: "[[A_PCM:.*outputs.*A-.*\.pcm]]" +// PCH-NEXT: "{{.*}}[[A_PCM:A-.*\.pcm]]" // PCH: "-fcache-compile-job" // PCH: "-emit-module" // PCH: "-fmodule-file-cache-key" -// PCH: "[[B_PCM:.*outputs.*B-.*\.pcm]]" +// PCH: "[[B_PCM:B-.*\.pcm]]" // PCH: "[[B_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // PCH: "-fmodule-file={{(B=)?}}[[B_PCM]]" // PCH: ] @@ -88,7 +88,7 @@ // PCH: "-fcas-fs" // PCH-NEXT: "[[B_ROOT_ID]]" // PCH: "-o" -// PCH-NEXT: "[[B_PCM]]" +// PCH-NEXT: "{{.*}}[[B_PCM]]" // PCH: "-fcache-compile-job" // PCH: "-emit-module" // PCH: ] @@ -140,10 +140,10 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[C_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[C_PCM:.*outputs.*C-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[C_PCM:C-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file={{(B=)?}}[[B_PCM:.*outputs.*B-.*\.pcm]]" +// CHECK: "-fmodule-file={{(B=)?}}[[B_PCM:B-.*\.pcm]]" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[B_PCM]]" // CHECK: "[[B_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" diff --git a/clang/test/ClangScanDeps/modules-cas-trees.c b/clang/test/ClangScanDeps/modules-cas-trees.c index 2077b697c6c13..1d1bd1de839d1 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees.c +++ b/clang/test/ClangScanDeps/modules-cas-trees.c @@ -78,11 +78,11 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[LEFT_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[LEFT_PCM:.*outputs.*Left-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[LEFT_PCM:Left-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[TOP_PCM:.*outputs.*Top-.*\.pcm]]" +// CHECK: "{{.*}}[[TOP_PCM:Top-.*\.pcm]]" // CHECK: "[[TOP_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "-fmodule-file={{(Top=)?}}[[TOP_PCM]]" // CHECK: ] @@ -107,7 +107,7 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[RIGHT_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[RIGHT_PCM:.*outputs.*Right-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[RIGHT_PCM:Right-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" @@ -132,7 +132,7 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[TOP_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[TOP_PCM]]" +// CHECK-NEXT: "{{.*}}[[TOP_PCM]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c b/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c index fa5c123975401..084b0f5837fd0 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c +++ b/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c @@ -25,7 +25,7 @@ // DOT: digraph "dependencies" // DOT-DAG: [[TU:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}tu.m"]; // DOT-DAG: [[HEADER:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}A.h"]; -// DOT-DAG: [[PCM:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}Mod-{{.*}}.pcm"]; +// DOT-DAG: [[PCM:header_[0-9]+]] [ shape="box", label="Mod-{{.*}}.pcm"]; // DOT-DAG: [[TU]] -> [[HEADER]] // DOT-DAG: [[HEADER]] -> [[PCM]] diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c new file mode 100644 index 0000000000000..f15c4244f2fd8 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c @@ -0,0 +1,192 @@ +// Test that modifications to a common header (imported from both a PCH and a TU) +// cause rebuilds of dependent modules imported from the TU on incremental build. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +//--- module.modulemap +module mod_common { header "mod_common.h" } +module mod_tu { header "mod_tu.h" } +module mod_tu_extra { header "mod_tu_extra.h" } + +//--- mod_common.h +#define MOD_COMMON_MACRO 0 +//--- mod_tu.h +#include "mod_common.h" +#if MOD_COMMON_MACRO +#include "mod_tu_extra.h" +#endif + +//--- mod_tu_extra.h + +//--- prefix.h +#include "mod_common.h" + +//--- tu.c +#include "mod_tu.h" + +// Clean: scan the PCH. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_pch_clean.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -x c-header %t/prefix.h -o %t/prefix.h.pch -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache + +// Clean: build the PCH. +// RUN: %deps-to-rsp %t/deps_pch_clean.json --module-name mod_common > %t/mod_common.rsp +// RUN: %deps-to-rsp %t/deps_pch_clean.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/mod_common.rsp +// RUN: %clang @%t/pch.rsp + +// Clean: scan the TU. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_clean.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// RUN: cat %t/deps_tu_clean.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-CLEAN -DPREFIX=%/t +// CHECK-TU-CLEAN: { +// CHECK-TU-CLEAN-NEXT: "modules": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-module-deps": [], +// CHECK-TU-CLEAN-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-CLEAN-NEXT: "command-line": [ +// CHECK-TU-CLEAN: ], +// CHECK-TU-CLEAN-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "file-deps": [ +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/mod_tu.h" +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "link-libraries": [], +// CHECK-TU-CLEAN-NEXT: "name": "mod_tu" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "translation-units": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "commands": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-module-deps": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "module-name": "mod_tu" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "command-line": [ +// CHECK-TU-CLEAN: ], +// CHECK-TU-CLEAN-NEXT: "executable": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "file-deps": [ +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/tu.c", +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "input-file": "[[PREFIX]]/tu.c" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ] +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN: ] +// CHECK-TU-CLEAN: } + +// Clean: build the TU. +// RUN: %deps-to-rsp %t/deps_tu_clean.json --module-name mod_tu > %t/mod_tu.rsp +// RUN: %deps-to-rsp %t/deps_tu_clean.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/mod_tu.rsp +// RUN: %clang @%t/tu.rsp + +// Incremental: modify the common module. +// RUN: sleep 1 +// RUN: echo "#define MOD_COMMON_MACRO 1" > %t/mod_common.h +// RUN: echo "#define MOD_COMMON_MACRO_NEW 1" >> %t/mod_common.h + +// Incremental: scan the PCH. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_pch_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -x c-header %t/prefix.h -o %t/prefix.h.pch -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache + +// Incremental: build the PCH. +// RUN: %deps-to-rsp %t/deps_pch_incremental.json --module-name mod_common > %t/mod_common.rsp +// RUN: %deps-to-rsp %t/deps_pch_incremental.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/mod_common.rsp +// RUN: %clang @%t/pch.rsp + +// Incremental: scan the TU. This needs to invalidate modules imported from the +// TU that depend on modules imported from the PCH and discover the +// new dependency on 'mod_tu_extra'. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// RUN: cat %t/deps_tu_incremental.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-INCREMENTAL -DPREFIX=%/t +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// CHECK-TU-INCREMENTAL: { +// CHECK-TU-INCREMENTAL-NEXT: "modules": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "module-name": "mod_tu_extra" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/mod_tu.h" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "link-libraries": [], +// CHECK-TU-INCREMENTAL-NEXT: "name": "mod_tu" +// CHECK-TU-INCREMENTAL-NEXT: }, +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [], +// CHECK-TU-INCREMENTAL-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/mod_tu_extra.h" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "link-libraries": [], +// CHECK-TU-INCREMENTAL-NEXT: "name": "mod_tu_extra" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "translation-units": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "commands": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "module-name": "mod_tu" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "executable": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/tu.c", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "input-file": "[[PREFIX]]/tu.c" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ] +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL: ] +// CHECK-TU-INCREMENTAL: } + +// Incremental: build the TU. +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --module-name mod_tu_extra > %t/mod_tu_extra.rsp +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --module-name mod_tu > %t/mod_tu.rsp +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/mod_tu_extra.rsp +// RUN: %clang @%t/mod_tu.rsp +// RUN: %clang @%t/tu.rsp diff --git a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c index 8eb7da7be4bea..46531cf8c1e89 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c +++ b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c @@ -132,9 +132,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" // CHECK: "-fno-implicit-modules" @@ -167,9 +167,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key -// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" // CHECK: "-fno-implicit-modules" @@ -261,13 +261,13 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "Left-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Left=/^modules/{{.*}}/Left-{{.*}}.pcm" -// CHECK: "-fmodule-file=Right=/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Left=Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c index d8efe277c35ab..656f46c61856b 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -53,9 +53,9 @@ // CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -91,11 +91,11 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-x" // CHECK-NEXT: "c" -// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index de61673523e53..63d7af7bbec4f 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -116,9 +116,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" // CHECK: "-fno-implicit-modules" @@ -151,9 +151,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" // CHECK: "-fno-implicit-modules" @@ -273,13 +273,13 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "Left-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" -// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Left=Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c b/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c index aee657ce58e3d..0572d2bb4a0df 100644 --- a/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c +++ b/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c @@ -96,7 +96,7 @@ // PCH: "-o" // PCH: "[[PREFIX]]/modules/{{.*}}/A-{{.*}}.pcm" // PCH: "-fmodule-file-cache-key" -// PCH: "/^modules/{{.*}}/B-[[B_CONTEXT_HASH:[^.]+]].pcm" +// PCH: "B-[[B_CONTEXT_HASH:[^.]+]].pcm" // PCH: "llvmcas://{{.*}}" // PCH: "-x" // PCH: "c" @@ -105,7 +105,7 @@ // PCH: "/^sdk" // PCH: "-resource-dir" // PCH: "/^tc/lib/clang/{{.*}}" -// PCH: "-fmodule-file=B=/^modules/{{.*}}/B-[[B_CONTEXT_HASH]].pcm" +// PCH: "-fmodule-file=B=B-[[B_CONTEXT_HASH]].pcm" // PCH: "-isystem" // PCH: "/^tc/lib/clang/{{.*}}/include" // PCH: "-internal-externc-isystem" @@ -172,7 +172,7 @@ // PCH: "/^src" // PCH: "-fmodule-map-file=/^src/module.modulemap" // PCH: "-fmodule-file-cache-key" -// PCH: "/^modules/{{.*}}A-{{.*}}.pcm" +// PCH: "{{.*}}A-{{.*}}.pcm" // PCH: "llvmcas://{{.*}}" // PCH: "-x" // PCH: "c-header" @@ -181,7 +181,7 @@ // PCH: "/^sdk" // PCH: "-resource-dir" // PCH: "/^tc/lib/clang/{{.*}}" -// PCH: "-fmodule-file=A=/^modules/{{.*}}/A-{{.*}}.pcm" +// PCH: "-fmodule-file=A=A-{{.*}}.pcm" // PCH: "-isystem" // PCH: "/^tc/lib/clang/{{.*}}/include" // PCH: "-internal-externc-isystem" @@ -208,9 +208,9 @@ // CHECK-NEXT: "/^src" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file=/^modules/{{.*}}/B-{{.*}}.pcm" +// CHECK: "-fmodule-file=B-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}/B-{{.*}}.pcm" +// CHECK: "B-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -244,7 +244,7 @@ // CHECK-NEXT: "/^src" // CHECK: "-fmodule-map-file=/^src/module.modulemap" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}C-{{.*}}.pcm" +// CHECK: "C-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -253,7 +253,7 @@ // CHECK-NEXT: "/^sdk" // CHECK: "-resource-dir" // CHECK-NEXT: "/^tc/lib/clang/{{.*}}" -// CHECK: "-fmodule-file=C=/^modules/{{.*}}/C-{{.*}}.pcm" +// CHECK: "-fmodule-file=C=C-{{.*}}.pcm" // CHECK: "-isystem" // CHECK-NEXT: "/^sdk/usr/local/include" // CHECK: "-isystem" diff --git a/llvm/tools/dsymutil/BinaryHolder.cpp b/llvm/tools/dsymutil/BinaryHolder.cpp index 58751bddd3ba7..ed226681aea74 100644 --- a/llvm/tools/dsymutil/BinaryHolder.cpp +++ b/llvm/tools/dsymutil/BinaryHolder.cpp @@ -42,8 +42,9 @@ getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem, } BinaryHolder::BinaryHolder(IntrusiveRefCntPtr VFS, - BinaryHolder::Options Opts) - : VFS(VFS), Opts(Opts) {} + BinaryHolder::Options Opts, + std::shared_ptr CAS) + : VFS(VFS), CAS(std::move(CAS)), Opts(Opts) {} Error BinaryHolder::ArchiveEntry::load(IntrusiveRefCntPtr VFS, StringRef Filename, @@ -144,6 +145,37 @@ Error BinaryHolder::ObjectEntry::load(IntrusiveRefCntPtr VFS, return Error::success(); } +bool BinaryHolder::ObjectEntry::load(cas::ObjectStore &CAS, + StringRef PossibleID, Options Opts) { + auto ID = CAS.parseID(PossibleID); + if (!ID) { + consumeError(ID.takeError()); + return false; + } + + if (Opts.Verbose) + WithColor::note() << "try load cas object " << PossibleID << "\n"; + + auto Ref = CAS.getProxy(*ID); + if (!Ref) { + WithColor::warning() << "failed to load CAS object " << PossibleID << "\n"; + return false; + } + + // Only module files should be loaded here. Always not fat. + auto ObjectBuf = Ref->getMemoryBuffer(); + auto ErrOrObjectFile = + object::ObjectFile::createObjectFile(ObjectBuf->getMemBufferRef()); + if (!ErrOrObjectFile) { + WithColor::warning() << "malformed object loaded via " << PossibleID + << "\n"; + return false; + } + + Objects.push_back(std::move(*ErrOrObjectFile)); + return true; +} + std::vector BinaryHolder::ObjectEntry::getObjects() const { std::vector Result; @@ -265,9 +297,11 @@ BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { ObjectRefCounter[Filename]++; if (!ObjectCache.count(Filename)) { auto OE = std::make_unique(); - auto Err = OE->load(VFS, Filename, Timestamp, Opts); - if (Err) - return std::move(Err); + if (!(CAS && OE->load(*CAS, Filename, Opts))) { + auto Err = OE->load(VFS, Filename, Timestamp, Opts); + if (Err) + return std::move(Err); + } ObjectCache[Filename] = std::move(OE); } diff --git a/llvm/tools/dsymutil/BinaryHolder.h b/llvm/tools/dsymutil/BinaryHolder.h index cb5bd95978144..6a438deaa4efe 100644 --- a/llvm/tools/dsymutil/BinaryHolder.h +++ b/llvm/tools/dsymutil/BinaryHolder.h @@ -15,6 +15,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Object/Archive.h" #include "llvm/Object/Error.h" #include "llvm/Object/MachOUniversal.h" @@ -46,7 +47,8 @@ class BinaryHolder { }; BinaryHolder(IntrusiveRefCntPtr VFS, - BinaryHolder::Options Opts = {}); + BinaryHolder::Options Opts = {}, + std::shared_ptr CAS = nullptr); // Forward declarations for friend declaration. class ObjectEntry; @@ -67,6 +69,9 @@ class BinaryHolder { Error load(IntrusiveRefCntPtr VFS, StringRef Filename, TimestampTy Timestamp, BinaryHolder::Options = {}); + bool load(cas::ObjectStore &CAS, StringRef Filename, + BinaryHolder::Options = {}); + /// Access all owned ObjectFiles. std::vector getObjects() const; @@ -150,6 +155,8 @@ class BinaryHolder { /// Virtual File System instance. IntrusiveRefCntPtr VFS; + std::shared_ptr CAS; + Options Opts; }; diff --git a/llvm/tools/dsymutil/Options.td b/llvm/tools/dsymutil/Options.td index ad35e55e33b12..31ad0ade4fb3b 100644 --- a/llvm/tools/dsymutil/Options.td +++ b/llvm/tools/dsymutil/Options.td @@ -218,3 +218,8 @@ def dsym_search_path: Separate<["-", "--"], "D">, MetaVarName<"">, HelpText<"Specify a directory that contain dSYM files to search for.">, Group; + +def cas : Separate<["-", "--"], "cas">, + MetaVarName<"">, + HelpText<"Specify CAS directory to load CAS references from.">, + Group; diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp index 913077eb0b06d..b77c946f337b3 100644 --- a/llvm/tools/dsymutil/dsymutil.cpp +++ b/llvm/tools/dsymutil/dsymutil.cpp @@ -23,6 +23,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/BuiltinUnifiedCASDatabases.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/DWARF/DWARFVerifier.h" @@ -120,6 +123,7 @@ struct DsymutilOptions { DWARFVerify Verify = DWARFVerify::Default; ReproducerMode ReproMode = ReproducerMode::GenerateOnCrash; dsymutil::LinkOptions LinkOpts; + std::string CASPath; }; /// Return a list of input files. This function has logic for dealing with the @@ -397,6 +401,9 @@ static Expected getOptions(opt::InputArgList &Args) { for (auto *SearchPath : Args.filtered(OPT_dsym_search_path)) Options.LinkOpts.DSYMSearchPaths.push_back(SearchPath->getValue()); + if (opt::Arg *CASPath = Args.getLastArg(OPT_cas)) + Options.CASPath = CASPath->getValue(); + if (Error E = verifyOptions(Options)) return std::move(E); return Options; @@ -666,12 +673,24 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) { return EXIT_FAILURE; } + // Create CAS. Only support unified database for now. + std::shared_ptr CAS; + if (!Options.CASPath.empty()) { + auto DB = cas::createOnDiskUnifiedCASDatabases(Options.CASPath); + if (!DB) { + WithColor::error() << "failed to open CAS: " << toString(DB.takeError()) + << "\n"; + return EXIT_FAILURE; + } + CAS = std::move(DB->first); + } + for (auto &InputFile : Options.InputFiles) { // Shared a single binary holder for all the link steps. BinaryHolder::Options BinOpts; BinOpts.Verbose = Options.LinkOpts.Verbose; BinOpts.Warn = !Options.NoObjectTimestamp; - BinaryHolder BinHolder(Options.LinkOpts.VFS, BinOpts); + BinaryHolder BinHolder(Options.LinkOpts.VFS, BinOpts, CAS); // Dump the symbol table for each input file and requested arch if (Options.DumpStab) { From 407e665aec522d6a0dfdab6d4d7985504df28b0f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 24 Jul 2025 13:29:19 -0700 Subject: [PATCH 3/8] dsymutil auto infer cas-path --- clang/test/ClangScanDeps/gmodules.c | 13 ++++++++++++- llvm/tools/dsymutil/BinaryHolder.cpp | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c index 6f5c827e4f246..674a6aa671435 100644 --- a/clang/test/ClangScanDeps/gmodules.c +++ b/clang/test/ClangScanDeps/gmodules.c @@ -4,6 +4,7 @@ // RUN: split-file %s %t // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json // RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json +// RUN: sed "s|DIR|%/t|g" %t/cas-config.template > %t/.cas-config /// Scan PCH // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ @@ -36,6 +37,7 @@ /// Check debug info is correct. // RUN: %clang %t/tu.o -o %t/a.out // RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty +// RUN: dsymutil %t/a.out -o %t/a2.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty // WARN-NOT: warning: // RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF @@ -58,7 +60,11 @@ // RUN: %deps-to-rsp %t/deps_2.json --module-name Right > %t/Right_2.rsp // RUN: %deps-to-rsp %t/deps_2.json --tu-index 0 > %t/tu_2.rsp // RUN: %clang @%t/Right_2.rsp -// RUN: %clang @%t/tu_2.rsp +// RUN: %clang @%t/tu_2.rsp -o %t/tu_2.o + +/// Diff all outputs +// RUN: diff %t/prefix.h.pch %t/prefix_2.pch +// RUN: diff %t/tu.o %t/tu_2.o // Check all the types are available // DWARF: DW_TAG_compile_unit @@ -213,3 +219,8 @@ int main(void) { struct Top _top; struct Prefix _prefix; } + +//--- cas-config.template +{ + "CASPath": "DIR/cas" +} diff --git a/llvm/tools/dsymutil/BinaryHolder.cpp b/llvm/tools/dsymutil/BinaryHolder.cpp index ed226681aea74..af04248fac226 100644 --- a/llvm/tools/dsymutil/BinaryHolder.cpp +++ b/llvm/tools/dsymutil/BinaryHolder.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "BinaryHolder.h" +#include "llvm/CAS/CASConfiguration.h" #include "llvm/Object/MachO.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" @@ -263,6 +264,22 @@ BinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename, return *(MemberCache[Key] = std::move(OE)); } +static std::shared_ptr searchAndCreateCAS(StringRef Path) { + auto Config = cas::CASConfiguration::createFromSearchConfigFile(Path); + if (!Config) + return nullptr; + + auto DB = Config->second.createDatabases(); + if (!DB) { + consumeError(DB.takeError()); + return nullptr; + } + + WithColor::note() << "create CAS using configuration: " << Config->first + << "'\n"; + return DB->first; +} + Expected BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { if (Opts.Verbose) @@ -297,6 +314,9 @@ BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { ObjectRefCounter[Filename]++; if (!ObjectCache.count(Filename)) { auto OE = std::make_unique(); + if (!CAS) + CAS = searchAndCreateCAS(Filename); + if (!(CAS && OE->load(*CAS, Filename, Opts))) { auto Err = OE->load(VFS, Filename, Timestamp, Opts); if (Err) From e2580b886189cebcd7efa09d1baee8ec2b39e1c5 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 4 Aug 2025 15:40:56 -0700 Subject: [PATCH 4/8] CAS configuration --- lldb/include/lldb/Core/ModuleList.h | 6 ++++++ lldb/source/Core/CoreProperties.td | 13 +++++++++++++ lldb/source/Core/ModuleList.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h index 1109572e27d21..9ee7e1e79a46d 100644 --- a/lldb/include/lldb/Core/ModuleList.h +++ b/lldb/include/lldb/Core/ModuleList.h @@ -100,6 +100,12 @@ class ModuleListProperties : public Properties { bool GetSwiftEnableASTContext() const; // END SWIFT + // START CAS + FileSpec GetCASOnDiskPath() const; + FileSpec GetCASPluginPath() const; + std::vector> GetCASPluginOptions() const; + // END CAS + FileSpec GetClangModulesCachePath() const; bool SetClangModulesCachePath(const FileSpec &path); bool GetEnableExternalLookup() const; diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td index 2ebff3d4e28a1..2e511bf8a20ca 100644 --- a/lldb/source/Core/CoreProperties.td +++ b/lldb/source/Core/CoreProperties.td @@ -74,6 +74,19 @@ let Definition = "modulelist" in { DefaultTrue, Desc<"Enable instantiating Swift AST contexts.">; // END SWIFT +// BEGIN CAS + def CASOnDiskPath: Property<"cas-path", "FileSpec">, + Global, + DefaultStringValue<"">, + Desc<"The path for CAS storage">; + def CASPluginPath: Property<"cas-plugin-path", "FileSpec">, + Global, + DefaultStringValue<"">, + Desc<"The path for CAS plugin dynamic library">; + def CASPluginOptions: Property<"cas-plugin-options", "Array">, + ElementType<"String">, + Desc<"A list of options to be passed to CAS plugins">; +// END CAS def SymLinkPaths: Property<"debug-info-symlink-paths", "FileSpecList">, Global, DefaultStringValue<"">, diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp index 7e4047b75cf3f..97f486a5104bf 100644 --- a/lldb/source/Core/ModuleList.cpp +++ b/lldb/source/Core/ModuleList.cpp @@ -282,6 +282,32 @@ bool ModuleListProperties::GetSwiftEnableASTContext() const { } // END SWIFT +// START CAS +FileSpec ModuleListProperties::GetCASOnDiskPath() const { + const uint32_t idx = ePropertyCASOnDiskPath; + return GetPropertyAtIndexAs(idx, {}); +} + +FileSpec ModuleListProperties::GetCASPluginPath() const { + const uint32_t idx = ePropertyCASPluginPath; + return GetPropertyAtIndexAs(idx, {}); +} + +std::vector> +ModuleListProperties::GetCASPluginOptions() const { + Args args; + const uint32_t idx = ePropertyCASPluginOptions; + m_collection_sp->GetPropertyAtIndexAsArgs(idx, args); + std::vector> options; + for (auto &arg: args) { + llvm::StringRef opt = arg.c_str(); + auto splitted = opt.split("="); + options.emplace_back(splitted.first.str(), splitted.second.str()); + } + return options; +} +// END CAS + FileSpec ModuleListProperties::GetLLDBIndexCachePath() const { const uint32_t idx = ePropertyLLDBIndexCachePath; return GetPropertyAtIndexAs(idx, {}); From 6e59c4a718a66d45a37b221834890c9f5013314c Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 5 Aug 2025 10:02:22 -0700 Subject: [PATCH 5/8] [lldb] Teach LLDB to load gmodules from CAS --- clang/test/ClangScanDeps/gmodules.c | 1 + lldb/include/lldb/Core/ModuleList.h | 5 + lldb/source/Core/ModuleList.cpp | 107 ++++++++++++++++++ .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 13 ++- 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c index 674a6aa671435..52cc8acb57dd1 100644 --- a/clang/test/ClangScanDeps/gmodules.c +++ b/clang/test/ClangScanDeps/gmodules.c @@ -218,6 +218,7 @@ int main(void) { struct Right _right; struct Top _top; struct Prefix _prefix; + return 0; } //--- cas-config.template diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h index 9ee7e1e79a46d..1f2873ab3543b 100644 --- a/lldb/include/lldb/Core/ModuleList.h +++ b/lldb/include/lldb/Core/ModuleList.h @@ -528,6 +528,11 @@ class ModuleList { llvm::SmallVectorImpl *old_modules, bool *did_create_ptr, bool always_create = false); + static Status GetSharedModuleFromCAS(ConstString module_name, + const char *cas_id, FileSpec cu_path, + ModuleSpec &module_spec, + lldb::ModuleSP &module_sp); + static bool RemoveSharedModule(lldb::ModuleSP &module_sp); static void FindSharedModules(const ModuleSpec &module_spec, diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp index 97f486a5104bf..017a70ea306d9 100644 --- a/lldb/source/Core/ModuleList.cpp +++ b/lldb/source/Core/ModuleList.cpp @@ -26,6 +26,7 @@ #include "lldb/Utility/Log.h" #include "lldb/Utility/UUID.h" #include "lldb/lldb-defines.h" +#include "llvm/Support/FileUtilities.h" #if defined(_WIN32) #include "lldb/Host/windows/PosixApi.h" @@ -33,6 +34,8 @@ #include "clang/Driver/Driver.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CAS/CASConfiguration.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Threading.h" #include "llvm/Support/raw_ostream.h" @@ -1050,8 +1053,13 @@ namespace { struct SharedModuleListInfo { ModuleList module_list; ModuleListProperties module_list_properties; + std::shared_ptr cas_object_store; + llvm::StringMap loaded_cas_files; + std::vector> module_file_removers; + std::mutex shared_lock; }; } + static SharedModuleListInfo &GetSharedModuleListInfo() { static SharedModuleListInfo *g_shared_module_list_info = nullptr; @@ -1070,6 +1078,45 @@ static ModuleList &GetSharedModuleList() { return GetSharedModuleListInfo().module_list; } +static std::shared_ptr +GetOrCreateCASStorage(FileSpec CandidateConfigSearchPath) { + auto &shared_module_list = GetSharedModuleListInfo(); + if (shared_module_list.cas_object_store) + return shared_module_list.cas_object_store; + + std::scoped_lock lock(shared_module_list.shared_lock); + // Config CAS from properties. + llvm::cas::CASConfiguration cas_config; + cas_config.CASPath = + ModuleList::GetGlobalModuleListProperties().GetCASOnDiskPath().GetPath(); + cas_config.PluginPath = + ModuleList::GetGlobalModuleListProperties().GetCASPluginPath().GetPath(); + cas_config.PluginOptions = + ModuleList::GetGlobalModuleListProperties().GetCASPluginOptions(); + + if (!cas_config.CASPath.empty()) { + if (auto maybe_cas = cas_config.createDatabases()) { + shared_module_list.cas_object_store = std::move(maybe_cas->first); + return shared_module_list.cas_object_store; + } else + llvm::consumeError(maybe_cas.takeError()); + } + + // Try search from candidiate path. + auto search_config = llvm::cas::CASConfiguration::createFromSearchConfigFile( + CandidateConfigSearchPath.GetPath()); + if (!search_config) + return nullptr; + + if (auto maybe_cas = search_config->second.createDatabases()) { + shared_module_list.cas_object_store = std::move(maybe_cas->first); + return shared_module_list.cas_object_store; + } else + llvm::consumeError(maybe_cas.takeError()); + + return nullptr; +} + ModuleListProperties &ModuleList::GetGlobalModuleListProperties() { return GetSharedModuleListInfo().module_list_properties; } @@ -1342,6 +1389,66 @@ ModuleList::GetSharedModule(const ModuleSpec &module_spec, ModuleSP &module_sp, return error; } +static Status loadModuleFromCAS(ConstString module_name, const char *cas_id, + FileSpec cu_path, ModuleSpec &module_spec) { + auto cas = GetOrCreateCASStorage(cu_path); + if (!cas) + return Status::FromErrorString("CAS is not available"); + + auto &shared_module_list = GetSharedModuleListInfo(); + std::scoped_lock lock(shared_module_list.shared_lock); + auto result = shared_module_list.loaded_cas_files.find(cas_id); + if (result != shared_module_list.loaded_cas_files.end()) { + module_spec.GetFileSpec() = result->second; + return Status(); + } + + auto id = cas->parseID(cas_id) ; + if (!id) + return Status::FromError(id.takeError()); + + auto module_proxy = cas->getProxy(*id); + if (!module_proxy) + return Status::FromError(module_proxy.takeError()); + + // Create a temporary file that holds the module info. + // TODO: remove temporary file when finished. + llvm::SmallString<0> name; + int fd; + auto ec = + llvm::sys::fs::createTemporaryFile("lldb-module-cas", "pcm", fd, name); + if (ec || fd <= 0) + return Status::FromErrorString( + "Could not create temporary file for module"); + + NativeFile file(fd, File::eOpenOptionWriteOnly, true); + size_t num_bytes = module_proxy->getData().size(); + file.Write(module_proxy->getData().data(), num_bytes); + + Log *log = GetLog(LLDBLog::Modules); + if (log != nullptr) + LLDB_LOGF(log, "loading module '%s' using CASID '%s' from temp file: %s", + module_name.AsCString(), cas_id, name.c_str()); + + module_spec.GetFileSpec().SetFile(name, FileSpec::Style::native); + shared_module_list.loaded_cas_files.try_emplace(cas_id, + module_spec.GetFileSpec()); + + return Status(); +} + +Status ModuleList::GetSharedModuleFromCAS(ConstString module_name, + const char *cas_id, FileSpec cu_path, + ModuleSpec &module_spec, + lldb::ModuleSP &module_sp) { + Status load_result = + loadModuleFromCAS(module_name, cas_id, cu_path, module_spec); + if (load_result.Fail()) + return load_result; + + return GetSharedModule(module_spec, module_sp, nullptr, nullptr, nullptr); +} + bool ModuleList::RemoveSharedModule(lldb::ModuleSP &module_sp) { return GetSharedModuleList().Remove(module_sp); } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 4bb2431b6641e..6ab38ba203111 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -1998,6 +1998,17 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() { continue; ModuleSpec dwo_module_spec; + + dwo_module_spec.GetArchitecture() = + m_objfile_sp->GetModule()->GetArchitecture(); + + // Try load from CAS, if loaded, continue to next one. + if (ModuleList::GetSharedModuleFromCAS(const_name, dwo_path, + GetObjectFile()->GetFileSpec(), + dwo_module_spec, module_sp) + .Success()) + continue; + dwo_module_spec.GetFileSpec().SetFile(dwo_path, FileSpec::Style::native); if (dwo_module_spec.GetFileSpec().IsRelative()) { const char *comp_dir = @@ -2009,8 +2020,6 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() { dwo_module_spec.GetFileSpec().AppendPathComponent(dwo_path); } } - dwo_module_spec.GetArchitecture() = - m_objfile_sp->GetModule()->GetArchitecture(); // When LLDB loads "external" modules it looks at the presence of // DW_AT_dwo_name. However, when the already created module From 4dbd095387a3e22e602fccbba6661b5288840c6d Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 6 Aug 2025 10:31:25 -0700 Subject: [PATCH 6/8] [lldb][tests] Fix build with LLVM_ENABLE_SWIFT_SUPPORT=OFF Fix the unittest build when swift support is off. --- lldb/unittests/Core/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lldb/unittests/Core/CMakeLists.txt b/lldb/unittests/Core/CMakeLists.txt index 15c57f3a29577..9d11c02d9837c 100644 --- a/lldb/unittests/Core/CMakeLists.txt +++ b/lldb/unittests/Core/CMakeLists.txt @@ -1,3 +1,11 @@ +set(SWIFT_SOURCES + SwiftDemanglingPartsTest.cpp +) + +set(LLVM_OPTIONAL_SOURCES ${SWIFT_SOURCES}) +if (NOT LLDB_ENABLE_SWIFT_SUPPORT) + unset(SWIFT_SOURCES) +endif() add_lldb_unittest(LLDBCoreTests DebuggerTest.cpp @@ -14,9 +22,10 @@ add_lldb_unittest(LLDBCoreTests SourceLocationSpecTest.cpp SourceManagerTest.cpp TelemetryTest.cpp - SwiftDemanglingPartsTest.cpp UniqueCStringMapTest.cpp + ${SWIFT_SOURCES} + LINK_COMPONENTS Support Telemetry From 0173e49b1f7f4ed291c4f238b7ba513471c817a0 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 6 Aug 2025 11:49:34 -0700 Subject: [PATCH 7/8] [lldb] Add clang-explicit-module-build caching test for gmodule --- lldb/scripts/clang-explicit-module-build.py | 58 +++++++++++++++++++++ lldb/test/Shell/SymbolFile/cas-gmodule.test | 46 ++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100755 lldb/scripts/clang-explicit-module-build.py create mode 100644 lldb/test/Shell/SymbolFile/cas-gmodule.test diff --git a/lldb/scripts/clang-explicit-module-build.py b/lldb/scripts/clang-explicit-module-build.py new file mode 100755 index 0000000000000..555a790bd0d2c --- /dev/null +++ b/lldb/scripts/clang-explicit-module-build.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Usage: clang-explicit-module-build.py [-cas-path CAS_PATH] -- CLANG_COMMAND + +The script builds with clang explicit module. If `-cas-path` is used, clang +explicit module build is going to enable compilation caching. +""" + +import argparse +import json +import os +import subprocess +import sys + + +def main(): + argv = sys.argv[1:] + if "--" not in argv: + print("missing clang command") + exit(1) + dash_idx = argv.index("--") + clang_args = argv[dash_idx + 1:] + if len(clang_args) == 0: + print("empty clang command") + exit(1) + + parser = argparse.ArgumentParser() + parser.add_argument("-cas-path", required=False) + args = parser.parse_args(argv[:dash_idx]) + + clang_exe = clang_args[0] + clang_scan_deps = os.path.join( + os.path.dirname(clang_exe), "clang-scan-deps") + + scan_cmd = [clang_scan_deps] + if args.cas_path is None: + scan_cmd.extend(["-format", "experimental-full", "-o", "-", "--"]) + else: + scan_cmd.extend(["-format", "experimental-include-tree-full", + "-cas-path", args.cas_path, "-o", "-", "--"]) + scan_cmd.extend(clang_args) + scan_result = json.loads(subprocess.check_output(scan_cmd)) + + # build module: assuming modules in reverse dependency order. + for module in reversed(scan_result["modules"]): + cmd = [clang_exe] + module["command-line"] + print(*cmd) + subprocess.check_call(cmd) + + # build tu: assuming only one TU. + tu_cmd = [clang_exe] + \ + scan_result["translation-units"][0]["commands"][0]["command-line"] + print(*tu_cmd) + subprocess.check_call(tu_cmd) + + +if __name__ == "__main__": + main() diff --git a/lldb/test/Shell/SymbolFile/cas-gmodule.test b/lldb/test/Shell/SymbolFile/cas-gmodule.test new file mode 100644 index 0000000000000..d50779820ff54 --- /dev/null +++ b/lldb/test/Shell/SymbolFile/cas-gmodule.test @@ -0,0 +1,46 @@ +# REQUIRES: system-darwin +# RUN: rm -rf %t +# RUN: split-file %s %t +# RUN: sed "s|DIR|%/t|g" %t/cas-config.template > %t/.cas-config + +# RUN: %python %S/../../../scripts/clang-explicit-module-build.py -cas-path %t/cas -- %clang_host -c -g -o %t/test.o %t/tu.c -fmodules -gmodules -fmodules-cache-path=%t/module-cache +# RUN: %clang_host %t/test.o -o %t/main +# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s +# CHECK: loading module 'Bottom' using CASID +# CHECK: loading module 'Top' using CASID +# CHECK: top = (x = 1) + +//--- module.modulemap +module Top { header "Top.h" export *} +module Bottom { header "Bottom.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Bottom.h +#pragma once +#include "Top.h" +struct Bottom { struct Top top; }; + +//--- tu.c +#include "Bottom.h" + +int main(void) { + struct Bottom _bottom; + _bottom.top.x = 1; + + return 0; // BREAK HERE +} + +//--- cas-config.template +{ + "CASPath": "DIR/cas" +} + +//--- lldb.script +log enable lldb module +b tu.c:7 +run +expr _bottom +quit From e3abb5df2e37dab204c2b13b8f47303d10ab72fc Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 11 Aug 2025 13:43:03 -0700 Subject: [PATCH 8/8] [lldb][SwiftASTContext] Teach clang module loading from CAS Teach SwiftASTContext to load clang module dependencies from CAS instead of FileSystem. --- .../TypeSystem/Swift/SwiftASTContext.cpp | 145 ++++++++++++++---- .../TypeSystem/Swift/SwiftASTContext.h | 9 ++ lldb/test/Shell/Swift/caching.test | 24 +++ 3 files changed, 150 insertions(+), 28 deletions(-) diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp index af5a5aeaef664..6c01938a0c43a 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp @@ -913,6 +913,51 @@ static std::string GetClangModulesCacheProperty() { return std::string(path); } +static void ConfigureCASStorage(SwiftASTContext *m_ast_context, + FileSpec CandidateConfigSearchPath) { + // Config CAS from properties. + llvm::cas::CASConfiguration cas_config; + cas_config.CASPath = + ModuleList::GetGlobalModuleListProperties().GetCASOnDiskPath().GetPath(); + cas_config.PluginPath = + ModuleList::GetGlobalModuleListProperties().GetCASPluginPath().GetPath(); + cas_config.PluginOptions = + ModuleList::GetGlobalModuleListProperties().GetCASPluginOptions(); + + auto &m_description = m_ast_context->GetDescription(); + + if (!cas_config.CASPath.empty()) { + if (auto maybe_cas = cas_config.createDatabases()) { + m_ast_context->SetCASStorage(std::move(maybe_cas->first), + std::move(maybe_cas->second)); + m_ast_context->GetCASOptions().Config = cas_config; + LOG_PRINTF(GetLog(LLDBLog::Types), + "Setup CAS from module list properties with cas path: %s", + cas_config.CASPath.c_str()); + return; + } else + llvm::consumeError(maybe_cas.takeError()); + } + + // Try search from candidiate path. + auto search_config = llvm::cas::CASConfiguration::createFromSearchConfigFile( + CandidateConfigSearchPath.GetPath()); + if (!search_config) + return; + + if (auto maybe_cas = search_config->second.createDatabases()) { + m_ast_context->SetCASStorage(std::move(maybe_cas->first), + std::move(maybe_cas->second)); + m_ast_context->GetCASOptions().Config = search_config->second; + LOG_PRINTF(GetLog(LLDBLog::Types), "Setup CAS from cas config file: %s", + search_config->first.c_str()); + return; + } else + llvm::consumeError(maybe_cas.takeError()); + + return; +} + SwiftASTContext::ScopedDiagnostics::ScopedDiagnostics( swift::DiagnosticConsumer &consumer) : m_consumer(consumer), @@ -1930,41 +1975,81 @@ void SwiftASTContext::AddExtraClangCC1Args( return; } - // Clear module cache key and other CAS options to load modules from disk - // directly. - invocation.getFrontendOpts().ModuleCacheKeys.clear(); - invocation.getCASOpts() = clang::CASOptions(); - - // Ignore CAS info inside modules when loading. - invocation.getFrontendOpts().ModuleLoadIgnoreCAS = true; - // Add options to allow clang importer to do implicit module build. invocation.getLangOpts().ImplicitModules = true; invocation.getHeaderSearchOpts().ImplicitModuleMaps = true; invocation.getHeaderSearchOpts().ModuleCachePath = GetCompilerInvocation().getClangModuleCachePath().str(); - // Remove non-existing modules in a systematic way. - auto CheckFileExists = [&](const std::string &file) -> bool { - if (llvm::sys::fs::exists(file)) - return true; - std::string warn; - llvm::raw_string_ostream(warn) - << "Nonexistent explicit module file " << file; - AddDiagnostic(eSeverityWarning, warn); - return false; - }; - for (auto it = invocation.getHeaderSearchOpts().PrebuiltModuleFiles.begin(); - it != invocation.getHeaderSearchOpts().PrebuiltModuleFiles.end();) { - if (!CheckFileExists(it->second)) - it = invocation.getHeaderSearchOpts().PrebuiltModuleFiles.erase(it); - else - ++it; + bool use_cas_module = m_cas && m_action_cache; + if (use_cas_module) { + // Load from CAS. + invocation.getCASOpts().CASPath = GetCASOptions().Config.CASPath; + invocation.getCASOpts().PluginPath = GetCASOptions().Config.PluginPath; + invocation.getCASOpts().PluginOptions = GetCASOptions().Config.PluginOptions; + + // Check the module availability in CAS, if not, fallback to regular load. + auto CheckModuleInCAS = [&](const std::string &key) { + auto id = m_cas->parseID(key); + if (!id) { + llvm::consumeError(id.takeError()); + return false; + } + auto lookup = m_action_cache->get(*id); + if (!lookup) { + llvm::consumeError(lookup.takeError()); + return false; + } + return (bool)*lookup; + }; + + use_cas_module = llvm::all_of(invocation.getFrontendOpts().ModuleCacheKeys, + [&](const auto &entry) { + auto exist = CheckModuleInCAS(entry.second); + if (!exist) { + LOG_PRINTF( + GetLog(LLDBLog::Types), + "module '%s' cannot be load " + "from CAS using key: %s, fallback to " + "load from file system", + entry.first.c_str(), + entry.second.c_str()); + } + return exist; + }); + } + + if (!use_cas_module) { + // Clear module cache key and other CAS options to load modules from disk + // directly. + invocation.getFrontendOpts().ModuleCacheKeys.clear(); + invocation.getCASOpts() = clang::CASOptions(); + + // Ignore CAS info inside modules when loading. + invocation.getFrontendOpts().ModuleLoadIgnoreCAS = true; + + // Remove non-existing modules in a systematic way. + auto CheckFileExists = [&](const std::string &file) -> bool { + if (llvm::sys::fs::exists(file)) + return true; + std::string warn; + llvm::raw_string_ostream(warn) + << "Nonexistent explicit module file " << file; + AddDiagnostic(eSeverityWarning, warn); + return false; + }; + for (auto it = invocation.getHeaderSearchOpts().PrebuiltModuleFiles.begin(); + it != invocation.getHeaderSearchOpts().PrebuiltModuleFiles.end();) { + if (!CheckFileExists(it->second)) + it = invocation.getHeaderSearchOpts().PrebuiltModuleFiles.erase(it); + else + ++it; + } + invocation.getFrontendOpts().ModuleFiles.erase( + llvm::remove_if(invocation.getFrontendOpts().ModuleFiles, + [&](const auto &mod) { return !CheckFileExists(mod); }), + invocation.getFrontendOpts().ModuleFiles.end()); } - invocation.getFrontendOpts().ModuleFiles.erase( - llvm::remove_if(invocation.getFrontendOpts().ModuleFiles, - [&](const auto &mod) { return !CheckFileExists(mod); }), - invocation.getFrontendOpts().ModuleFiles.end()); invocation.generateCC1CommandLine( [&](const llvm::Twine &arg) { dest.push_back(arg.str()); }); @@ -2546,6 +2631,8 @@ SwiftASTContext::CreateInstance(lldb::LanguageType language, Module &module, } } + ConfigureCASStorage(swift_ast_sp.get(), module.GetFileSpec()); + // The serialized triple is the triple of the last binary // __swiftast section that was processed. Instead of relying on // the section contents order, we overwrite the triple in the @@ -3026,6 +3113,8 @@ lldb::TypeSystemSP SwiftASTContext::CreateInstance( } } + ConfigureCASStorage(swift_ast_sp.get(), sc.module_sp->GetFileSpec()); + std::string resource_dir = HostInfo::GetSwiftResourceDir( triple, swift_ast_sp->GetPlatformSDKPath()); ConfigureResourceDirs(swift_ast_sp->GetCompilerInvocation(), resource_dir, diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h index 9077565a96b29..ac941b4e38233 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h @@ -311,6 +311,12 @@ class SwiftASTContext : public TypeSystemSwift { m_platform_sdk_path = path.str(); } + void SetCASStorage(std::shared_ptr cas, + std::shared_ptr action_cache) { + m_cas = std::move(cas); + m_action_cache = std::move(action_cache); + } + /// \return the ExtraArgs of the ClangImporterOptions. const std::vector &GetClangArguments(); @@ -987,6 +993,9 @@ class SwiftASTContext : public TypeSystemSwift { library_load_cache; /// A cache for GetCompileUnitImports(); llvm::DenseSet> m_cu_imports; + /// ObjectStore and ActionCache; + std::shared_ptr m_cas; + std::shared_ptr m_action_cache; typedef std::map> ASTFileDataMap; ASTFileDataMap m_ast_file_data_map; diff --git a/lldb/test/Shell/Swift/caching.test b/lldb/test/Shell/Swift/caching.test index b8a8507b940af..2e0eaa5b4a670 100644 --- a/lldb/test/Shell/Swift/caching.test +++ b/lldb/test/Shell/Swift/caching.test @@ -9,8 +9,21 @@ # RUN: -module-name main -o %t/main # RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s + +## Setup CAS and try loading from CAS +# RUN: sed "s|DIR|%/t|g" %t/lldb.script.template > %t/lldb_2.script +# RUN: %lldb %t/main -s %t/lldb_2.script 2>&1 | FileCheck %s --check-prefix=CAS-LOAD --check-prefix=CHECK + +## Check fallback to file system. +# RUN: rm -rf %t/cas +# RUN: %lldb %t/main -s %t/lldb_2.script 2>&1 | FileCheck %s --check-prefix=CAS-FALLBACK --check-prefix=CHECK + +# CAS-FALLBACK: operator()() -- module '{{.*}}' cannot be load from CAS using key +# CAS-FALLBACK-SAME: fallback to load from file system +# CAS-LOAD: ConfigureCASStorage() -- Setup CAS from module list properties with cas path # CHECK: LogConfiguration() -- Extra clang arguments # CHECK-COUNT-1: LogConfiguration() -- -triple +# CAS-LOAD: LogConfiguration() -- -fmodule-file-cache-key # CHECK: (Int) ${{.*}} = 1 //--- main.swift @@ -28,3 +41,14 @@ run # Create a SwiftASTContext expr 1 quit + +//--- lldb.script.template +# Force loading from interface to simulate no binary module available. +settings set symbols.swift-module-loading-mode prefer-interface +settings set symbols.cas-path DIR/cas +log enable lldb types +b test +run +# Create a SwiftASTContext +expr 1 +quit