diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index 20fb4de6a2a73..422202caddfd4 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -114,6 +114,14 @@ struct ModuleDeps { /// Whether this is a "system" module. bool IsSystem; + /// Whether this module is fully composed of file & module inputs from + /// locations likely to stay the same across the active development and build + /// cycle. For example, when all those input paths only resolve in Sysroot. + /// + /// External paths, as opposed to virtual file paths, are always used + /// for computing this value. + bool IsInStableDirectories; + /// The path to the modulemap file which defines this module. /// /// This can be used to explicitly build this module. This file will @@ -219,6 +227,9 @@ class ModuleDepCollectorPP final : public PPCallbacks { llvm::DenseSet &AddedModules); void addAffectingClangModule(const Module *M, ModuleDeps &MD, llvm::DenseSet &AddedModules); + + /// Add discovered module dependency for the given module. + void addOneModuleDep(const Module *M, const ModuleID ID, ModuleDeps &MD); }; /// Collects modular and non-modular dependencies of the main file by attaching @@ -320,6 +331,13 @@ void resetBenignCodeGenOptions(frontend::ActionKind ProgramAction, const LangOptions &LangOpts, CodeGenOptions &CGOpts); +/// Determine if \c Input can be resolved within a stable directory. +/// +/// \param Directories Paths known to be in a stable location. e.g. Sysroot. +/// \param Input Path to evaluate. +bool isPathInStableDir(const ArrayRef Directories, + const StringRef Input); + } // end namespace dependencies } // end namespace tooling } // end namespace clang diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 36b75c1016cd8..1c36039efeae7 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -157,6 +157,32 @@ static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) { } } +/// Check a subset of invocation options to determine whether the current +/// context can safely be considered as stable. +static bool areOptionsInStableDir(CowCompilerInvocation &BuildInvocation, + const ArrayRef StableDirs) { + const auto &HSOpts = BuildInvocation.getHeaderSearchOpts(); + assert(isPathInStableDir(StableDirs, HSOpts.Sysroot) && + "Sysroots differ between module dependencies and current TU"); + + assert(isPathInStableDir(StableDirs, HSOpts.ResourceDir) && + "ResourceDirs differ between module dependencies and current TU"); + + for (const auto &Entry : HSOpts.UserEntries) { + if (!Entry.IgnoreSysRoot) + continue; + if (!isPathInStableDir(StableDirs, Entry.Path)) + return false; + } + + for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) { + if (!isPathInStableDir(StableDirs, SysPrefix.Prefix)) + return false; + } + + return true; +} + static std::vector splitString(std::string S, char Separator) { SmallVector Segments; StringRef(S).split(Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false); @@ -212,6 +238,25 @@ void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction, } } +bool dependencies::isPathInStableDir(const ArrayRef Directories, + const StringRef Input) { + auto PathStartsWith = [](StringRef Prefix, StringRef Path) { + auto PrefixIt = llvm::sys::path::begin(Prefix), + PrefixEnd = llvm::sys::path::end(Prefix); + for (auto PathIt = llvm::sys::path::begin(Path), + PathEnd = llvm::sys::path::end(Path); + PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) { + if (*PrefixIt != *PathIt) + return false; + } + return PrefixIt == PrefixEnd; + }; + + return any_of(Directories, [&](StringRef Dir) { + return !Dir.empty() && PathStartsWith(Dir, Input); + }); +} + static CowCompilerInvocation makeCommonInvocationForModuleBuild(CompilerInvocation CI) { CI.resetNonModularOptions(); @@ -698,6 +743,17 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ID.ModuleName = M->getFullModuleName(); MD.IsSystem = M->IsSystem; + + // Start off with the assumption that this module is shareable when there + // is a sysroot provided. As more dependencies are discovered, check if those + // come from the provided shared directories. + const llvm::SmallVector StableDirs = { + MDC.ScanInstance.getHeaderSearchOpts().Sysroot, + MDC.ScanInstance.getHeaderSearchOpts().ResourceDir}; + MD.IsInStableDirectories = + !StableDirs[0].empty() && + (llvm::sys::path::root_directory(StableDirs[0]) != StableDirs[0]); + // For modules which use export_as link name, the linked product that of the // corresponding export_as-named module. if (!M->UseExportAsModuleLinkName) @@ -739,6 +795,12 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MDC.ScanInstance.getASTReader()->visitInputFileInfos( *MF, /*IncludeSystem=*/true, [&](const serialization::InputFileInfo &IFI, bool IsSystem) { + if (MD.IsInStableDirectories) { + auto FullFilePath = ASTReader::ResolveImportedPath( + PathBuf, IFI.UnresolvedImportedFilename, MF->BaseDirectory); + MD.IsInStableDirectories = + isPathInStableDir(StableDirs, *FullFilePath); + } if (!(IFI.TopLevel && IFI.ModuleMap)) return; if (IFI.UnresolvedImportedFilenameAsRequested.ends_with( @@ -780,6 +842,11 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { } }); + // Check provided input paths from the invocation for determining + // IsInStableDirectories. + if (MD.IsInStableDirectories) + MD.IsInStableDirectories = areOptionsInStableDir(CI, StableDirs); + MDC.associateWithContextHash(CI, IgnoreCWD, MD); // Finish the compiler invocation. Requires dependencies and the context hash. @@ -821,8 +888,13 @@ void ModuleDepCollectorPP::addModulePrebuiltDeps( for (const Module *Import : M->Imports) if (Import->getTopLevelModule() != M->getTopLevelModule()) if (MDC.isPrebuiltModule(Import->getTopLevelModule())) - if (SeenSubmodules.insert(Import->getTopLevelModule()).second) + if (SeenSubmodules.insert(Import->getTopLevelModule()).second) { MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule()); + // Conservatively consider the module as not coming from stable + // directories, as transitive dependencies from the prebuilt module + // have not been determined. + MD.IsInStableDirectories = false; + } } void ModuleDepCollectorPP::addAllSubmoduleDeps( @@ -835,6 +907,13 @@ void ModuleDepCollectorPP::addAllSubmoduleDeps( }); } +void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID, + ModuleDeps &MD) { + MD.ClangModuleDeps.push_back(ID); + if (MD.IsInStableDirectories) + MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories; +} + void ModuleDepCollectorPP::addModuleDep( const Module *M, ModuleDeps &MD, llvm::DenseSet &AddedModules) { @@ -843,7 +922,7 @@ void ModuleDepCollectorPP::addModuleDep( !MDC.isPrebuiltModule(Import)) { if (auto ImportID = handleTopLevelModule(Import->getTopLevelModule())) if (AddedModules.insert(Import->getTopLevelModule()).second) - MD.ClangModuleDeps.push_back(*ImportID); + addOneModuleDep(Import->getTopLevelModule(), *ImportID, MD); } } } @@ -867,7 +946,7 @@ void ModuleDepCollectorPP::addAffectingClangModule( !MDC.isPrebuiltModule(Affecting)) { if (auto ImportID = handleTopLevelModule(Affecting)) if (AddedModules.insert(Affecting).second) - MD.ClangModuleDeps.push_back(*ImportID); + addOneModuleDep(Affecting, *ImportID, MD); } } } diff --git a/clang/test/ClangScanDeps/modules-in-stable-dirs.c b/clang/test/ClangScanDeps/modules-in-stable-dirs.c new file mode 100644 index 0000000000000..066c5445f41f4 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-in-stable-dirs.c @@ -0,0 +1,109 @@ +// This test verifies modules that are entirely comprised from stable directory inputs are captured in +// dependency information. + +// The first compilation verifies that transitive dependencies on local input are captured. +// The second compilation verifies that external paths are resolved when a +// vfsoverlay for determining is-in-stable-directories. + +// REQUIRES: shell +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json +// RUN: sed -e "s|DIR|%/t|g" %t/overlay.json.template > %t/overlay.json +// RUN: clang-scan-deps -compilation-database %t/compile-commands.json \ +// RUN: -j 1 -format experimental-full > %t/deps.db +// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t + +// CHECK: "modules": [ +// CHECK-NEXT: { +// CHECK: "is-in-stable-directories": true, +// CHECK: "name": "A" + +// Verify that there are no more occurances of sysroot. +// CHECK-NOT: "is-in-stable-directories" + +// CHECK: "name": "A" +// CHECK: "USE_VFS" +// CHECK: "name": "B" +// CHECK: "name": "C" +// CHECK: "name": "D" +// CHECK: "name": "NotInSDK" + +//--- compile-commands.json.in +[ +{ + "directory": "DIR", + "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps", + "file": "DIR/client.c" +}, +{ + "directory": "DIR", + "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -ivfsoverlay DIR/overlay.json -DUSE_VFS -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps", + "file": "DIR/client.c" +} +] + +//--- overlay.json.template +{ + "version": 0, + "case-sensitive": "false", + "roots": [ + { + "external-contents": "DIR/SysrootButNotReally/A/A_vfs.h", + "name": "DIR/Sysroot/usr/include/A/A_vfs.h", + "type": "file" + } + ] +} + +//--- Sysroot/usr/include/A/module.modulemap +module A { + umbrella "." +} + +//--- Sysroot/usr/include/A/A.h +#ifdef USE_VFS +#include +#endif +typedef int A_t; + +//--- SysrootButNotReally/A/A_vfs.h +typedef int typeFromVFS; + +//--- Sysroot/usr/include/B/module.modulemap +module B [system] { + umbrella "." +} + +//--- Sysroot/usr/include/B/B.h +#include +typedef int B_t; + +//--- Sysroot/usr/include/C/module.modulemap +module C [system] { + umbrella "." +} + +//--- Sysroot/usr/include/C/C.h +#include + +//--- Sysroot/usr/include/D/module.modulemap +module D [system] { + umbrella "." +} + +// Simulate a header that will be resolved in a local directory, from a sysroot header. +//--- Sysroot/usr/include/D/D.h +#include + +//--- BuildDir/module.modulemap +module NotInSDK [system] { + umbrella "." +} + +//--- BuildDir/HeaderNotFoundInSDK.h +typedef int local_t; + +//--- client.c +#include +#include diff --git a/clang/test/ClangScanDeps/prebuilt-modules-in-stable-dirs.c b/clang/test/ClangScanDeps/prebuilt-modules-in-stable-dirs.c new file mode 100644 index 0000000000000..910d4890a9072 --- /dev/null +++ b/clang/test/ClangScanDeps/prebuilt-modules-in-stable-dirs.c @@ -0,0 +1,114 @@ +/// This test validates that modules that depend on prebuilt modules resolve `is-in-stable-directories` as false. + +// REQUIRES: shell +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/overlay.json.template > %t/overlay.json +// RUN: sed -e "s|DIR|%/t|g" %t/compile-pch.json.in > %t/compile-pch.json +// RUN: clang-scan-deps -compilation-database %t/compile-pch.json \ +// RUN: -j 1 -format experimental-full > %t/deps_pch.db +// RUN: %clang -x c-header -c %t/prebuild.h -isysroot %t/MacOSX.sdk \ +// RUN: -I%t/BuildDir -ivfsoverlay %t/overlay.json \ +// RUN: -I %t/MacOSX.sdk/usr/include -fmodules -fmodules-cache-path=%t/module-cache \ +// RUN: -fimplicit-module-maps -o %t/prebuild.pch +// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json +// RUN: clang-scan-deps -compilation-database %t/compile-commands.json \ +// RUN: -j 1 -format experimental-full > %t/deps.db +// RUN: cat %t/deps_pch.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix PCH_DEP +// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t + +// PCH_DEP: "is-in-stable-directories": true +// PCH_DEP: "name": "A" + +// Verify is-in-stable-directories is not in any module dependencies, as they all depend on prebuilt modules. +// CHECK-NOT: "is-in-stable-directories" + +//--- compile-pch.json.in +[ +{ + "directory": "DIR", + "command": "clang -x c-header -c DIR/prebuild.h -isysroot DIR/MacOSX.sdk -IDIR/BuildDir -ivfsoverlay DIR/overlay.json -IDIR/MacOSX.sdk/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -o DIR/prebuild.pch", + "file": "DIR/prebuild.h" +} +] + +//--- compile-commands.json.in +[ +{ + "directory": "DIR", + "command": "clang -c DIR/client.c -isysroot DIR/MacOSX.sdk -IDIR/BuildDir -ivfsoverlay DIR/overlay.json -IDIR/MacOSX.sdk/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -include-pch DIR/prebuild.pch", + "file": "DIR/client.c" +} +] + +//--- overlay.json.template +{ + "version": 0, + "case-sensitive": "false", + "roots": [ + { + "external-contents": "DIR/BuildDir/B_vfs.h", + "name": "DIR/MacOSX.sdk/usr/include/B/B_vfs.h", + "type": "file" + } + ] +} + +//--- MacOSX.sdk/usr/include/A/module.modulemap +module A [system] { + umbrella "." +} + +//--- MacOSX.sdk/usr/include/A/A.h +typedef int A_type; + +//--- MacOSX.sdk/usr/include/B/module.modulemap +module B [system] { + umbrella "." +} + +//--- MacOSX.sdk/usr/include/B/B.h +#include + +//--- BuildDir/B_vfs.h +typedef int local_t; + +//--- MacOSX.sdk/usr/include/sys/sys.h +typedef int sys_t_m; + +//--- MacOSX.sdk/usr/include/sys/module.modulemap +module sys [system] { + umbrella "." +} + +//--- MacOSX.sdk/usr/include/B_transitive/B.h +#include + +//--- MacOSX.sdk/usr/include/B_transitive/module.modulemap +module B_transitive [system] { + umbrella "." +} + +//--- MacOSX.sdk/usr/include/C/module.modulemap +module C [system] { + umbrella "." +} + +//--- MacOSX.sdk/usr/include/C/C.h +#include + + +//--- MacOSX.sdk/usr/include/D/module.modulemap +module D [system] { + umbrella "." +} + +//--- MacOSX.sdk/usr/include/D/D.h +#include + +//--- prebuild.h +#include +#include // This dependency transitively depends on a local header. + +//--- client.c +#include // This dependency transitively depends on a local header. diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 3bdeb461e4bfa..5b255974cea15 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -471,6 +471,9 @@ class FullDeps { for (auto &&ModID : ModuleIDs) { auto &MD = Modules[ModID]; JOS.object([&] { + if (MD.IsInStableDirectories) + JOS.attribute("is-in-stable-directories", + MD.IsInStableDirectories); JOS.attributeArray("clang-module-deps", toJSONSorted(JOS, MD.ClangModuleDeps)); JOS.attribute("clang-modulemap-file",