Skip to content

Commit 4876b2f

Browse files
artemcmartemcm
committed
[Dependency Scanning] Bridge Clang dependency scanner results on-demand
Instead of always bridging all of the discovered modules of all of the queries, only do so for modules which are not already cached
1 parent 36246a2 commit 4876b2f

File tree

7 files changed

+211
-210
lines changed

7 files changed

+211
-210
lines changed

‎include/swift/AST/ModuleDependencies.h

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@
2323
#include "swift/Basic/Assertions.h"
2424
#include "swift/Basic/CXXStdlibKind.h"
2525
#include "swift/Basic/LLVM.h"
26+
#include "swift/ClangImporter/ClangImporter.h"
2627
#include "swift/Serialization/Validation.h"
2728
#include "clang/CAS/CASOptions.h"
2829
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
30+
#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
2931
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
3032
#include "llvm/ADT/ArrayRef.h"
3133
#include "llvm/ADT/DenseSet.h"
@@ -111,6 +113,18 @@ void registerCxxInteropLibraries(
111113
const llvm::Triple &Target, StringRef mainModuleName, bool hasStaticCxx,
112114
bool hasStaticCxxStdlib, CXXStdlibKind cxxStdlibKind,
113115
std::function<void(const LinkLibrary &)> RegistrationCallback);
116+
117+
using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
118+
using LookupModuleOutputCallback =
119+
llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
120+
clang::tooling::dependencies::ModuleOutputKind)>;
121+
122+
ModuleDependencyInfo
123+
bridgeClangModuleDependency(
124+
const ASTContext &ctx,
125+
const clang::tooling::dependencies::ModuleDeps &clangDependency,
126+
LookupModuleOutputCallback LookupModuleOutput,
127+
RemapPathCallback remapPath = nullptr);
114128
} // namespace dependencies
115129

116130
struct ScannerImportStatementInfo {
@@ -1107,8 +1121,11 @@ class ModuleDependenciesCache {
11071121
ModuleDependencyInfo dependencies);
11081122

11091123
/// Record dependencies for the given collection of Clang modules.
1110-
void recordClangDependencies(ModuleDependencyVector moduleDependencies,
1111-
DiagnosticEngine &diags);
1124+
void recordClangDependencies(
1125+
const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
1126+
const ASTContext &ctx,
1127+
dependencies::LookupModuleOutputCallback LookupModuleOutput,
1128+
dependencies::RemapPathCallback remapPath = nullptr);
11121129

11131130
/// Update stored dependencies for the given module.
11141131
void updateDependency(ModuleDependencyID moduleID,

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -492,19 +492,6 @@ class ClangImporter final : public ClangModuleLoader {
492492

493493
void verifyAllModules() override;
494494

495-
using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
496-
using LookupModuleOutputCallback =
497-
llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
498-
clang::tooling::dependencies::ModuleOutputKind)>;
499-
500-
static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
501-
bridgeClangModuleDependencies(
502-
const ASTContext &ctx,
503-
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
504-
clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
505-
LookupModuleOutputCallback LookupModuleOutput,
506-
RemapPathCallback remapPath = nullptr);
507-
508495
static void getBridgingHeaderOptions(
509496
const ASTContext &ctx,
510497
const clang::tooling::dependencies::TranslationUnitDeps &deps,

‎include/swift/DependencyScan/ModuleDependencyScanner.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ class ModuleDependencyScanningWorker {
4040

4141
private:
4242
/// Retrieve the module dependencies for the Clang module with the given name.
43-
ClangModuleScannerQueryResult scanFilesystemForClangModuleDependency(
43+
std::optional<clang::tooling::dependencies::TranslationUnitDeps>
44+
scanFilesystemForClangModuleDependency(
4445
Identifier moduleName,
4546
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
4647
&alreadySeenModules);

‎include/swift/Serialization/ScanningLoaders.h

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,6 @@ struct SwiftModuleScannerQueryResult {
4040
std::vector<IncompatibleCandidate> incompatibleCandidates;
4141
};
4242

43-
/// Result of looking up a Clang module on the current filesystem
44-
/// search paths.
45-
struct ClangModuleScannerQueryResult {
46-
ClangModuleScannerQueryResult(const ModuleDependencyVector &dependencyModuleGraph,
47-
const std::vector<std::string> &visibleModuleIdentifiers)
48-
: foundDependencyModuleGraph(dependencyModuleGraph),
49-
visibleModuleIdentifiers(visibleModuleIdentifiers) {}
50-
51-
ModuleDependencyVector foundDependencyModuleGraph;
52-
std::vector<std::string> visibleModuleIdentifiers;
53-
};
54-
5543
/// A module "loader" that looks for .swiftinterface and .swiftmodule files
5644
/// for the purpose of determining dependencies, but does not attempt to
5745
/// load the module files.

‎lib/AST/ModuleDependencies.cpp

Lines changed: 144 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "swift/AST/SourceFile.h"
2424
#include "swift/Frontend/Frontend.h"
2525
#include "swift/Strings.h"
26+
#include "clang/Lex/HeaderSearchOptions.h"
2627
#include "llvm/Config/config.h"
2728
#include "llvm/Support/Path.h"
2829
using namespace swift;
@@ -598,6 +599,120 @@ void swift::dependencies::registerCxxInteropLibraries(
598599
}
599600
}
600601

602+
ModuleDependencyInfo swift::dependencies::bridgeClangModuleDependency(
603+
const ASTContext &ctx,
604+
const clang::tooling::dependencies::ModuleDeps &clangModuleDep,
605+
LookupModuleOutputCallback lookupModuleOutput, RemapPathCallback callback) {
606+
auto remapPath = [&](StringRef path) {
607+
if (callback)
608+
return callback(path);
609+
return path.str();
610+
};
611+
612+
// File dependencies for this module.
613+
std::vector<std::string> fileDeps;
614+
clangModuleDep.forEachFileDep(
615+
[&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); });
616+
617+
std::vector<std::string> swiftArgs;
618+
auto addClangArg = [&](Twine arg) {
619+
swiftArgs.push_back("-Xcc");
620+
swiftArgs.push_back(arg.str());
621+
};
622+
623+
// We are using Swift frontend mode.
624+
swiftArgs.push_back("-frontend");
625+
626+
// Swift frontend action: -emit-pcm
627+
swiftArgs.push_back("-emit-pcm");
628+
swiftArgs.push_back("-module-name");
629+
swiftArgs.push_back(clangModuleDep.ID.ModuleName);
630+
631+
auto pcmPath = lookupModuleOutput(
632+
clangModuleDep,
633+
clang::tooling::dependencies::ModuleOutputKind::ModuleFile);
634+
swiftArgs.push_back("-o");
635+
swiftArgs.push_back(pcmPath);
636+
637+
// Ensure that the resulting PCM build invocation uses Clang frontend
638+
// directly
639+
swiftArgs.push_back("-direct-clang-cc1-module-build");
640+
641+
// Swift frontend option for input file path (Foo.modulemap).
642+
swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile));
643+
644+
auto invocation = clangModuleDep.getUnderlyingCompilerInvocation();
645+
// Clear some options from clang scanner.
646+
invocation.getMutFrontendOpts().ModuleCacheKeys.clear();
647+
invocation.getMutFrontendOpts().PathPrefixMappings.clear();
648+
invocation.getMutFrontendOpts().OutputFile.clear();
649+
650+
// Reset CASOptions since that should be coming from swift.
651+
invocation.getMutCASOpts() = clang::CASOptions();
652+
invocation.getMutFrontendOpts().CASIncludeTreeID.clear();
653+
654+
// FIXME: workaround for rdar://105684525: find the -ivfsoverlay option
655+
// from clang scanner and pass to swift.
656+
if (!ctx.CASOpts.EnableCaching) {
657+
auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles;
658+
for (auto overlay : overlayFiles) {
659+
swiftArgs.push_back("-vfsoverlay");
660+
swiftArgs.push_back(overlay);
661+
}
662+
}
663+
664+
// Add args reported by the scanner.
665+
auto clangArgs = invocation.getCC1CommandLine();
666+
llvm::for_each(clangArgs, addClangArg);
667+
668+
// CASFileSystemRootID.
669+
std::string RootID = clangModuleDep.CASFileSystemRootID
670+
? clangModuleDep.CASFileSystemRootID->toString()
671+
: "";
672+
673+
std::string IncludeTree =
674+
clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
675+
676+
ctx.CASOpts.enumerateCASConfigurationFlags(
677+
[&](StringRef Arg) { swiftArgs.push_back(Arg.str()); });
678+
679+
if (!IncludeTree.empty()) {
680+
swiftArgs.push_back("-clang-include-tree-root");
681+
swiftArgs.push_back(IncludeTree);
682+
}
683+
std::string mappedPCMPath = remapPath(pcmPath);
684+
685+
std::vector<LinkLibrary> LinkLibraries;
686+
for (const auto &ll : clangModuleDep.LinkLibraries)
687+
LinkLibraries.emplace_back(ll.Library,
688+
ll.IsFramework ? LibraryKind::Framework
689+
: LibraryKind::Library,
690+
/*static=*/false);
691+
692+
// Module-level dependencies.
693+
llvm::StringSet<> alreadyAddedModules;
694+
auto bridgedDependencyInfo = ModuleDependencyInfo::forClangModule(
695+
pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
696+
clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, LinkLibraries, RootID,
697+
IncludeTree, /*module-cache-key*/ "", clangModuleDep.IsSystem);
698+
699+
std::vector<ModuleDependencyID> directDependencyIDs;
700+
for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
701+
// FIXME: This assumes, conservatively, that all Clang module imports
702+
// are exported. We need to fix this once the clang scanner gains the
703+
// appropriate API to query this.
704+
bridgedDependencyInfo.addModuleImport(
705+
moduleName.ModuleName, /* isExported */ true, AccessLevel::Public,
706+
&alreadyAddedModules);
707+
// It is safe to assume that all dependencies of a Clang module are Clang
708+
// modules.
709+
directDependencyIDs.push_back(
710+
{moduleName.ModuleName, ModuleDependencyKind::Clang});
711+
}
712+
bridgedDependencyInfo.setImportedClangDependencies(directDependencyIDs);
713+
return bridgedDependencyInfo;
714+
}
715+
601716
void
602717
swift::dependencies::registerBackDeployLibraries(
603718
const IRGenOptions &IRGenOpts,
@@ -779,16 +894,19 @@ void ModuleDependenciesCache::recordDependency(
779894
}
780895

781896
void ModuleDependenciesCache::recordClangDependencies(
782-
ModuleDependencyVector dependencies, DiagnosticEngine &diags) {
897+
const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
898+
const ASTContext &ctx,
899+
dependencies::LookupModuleOutputCallback lookupModuleOutput,
900+
dependencies::RemapPathCallback remapPath) {
783901
for (const auto &dep : dependencies) {
784-
ASSERT(dep.first.Kind == ModuleDependencyKind::Clang);
785-
auto newClangModuleDetails = dep.second.getAsClangModule();
786-
if (hasDependency(dep.first)) {
902+
auto depID =
903+
ModuleDependencyID{dep.ID.ModuleName, ModuleDependencyKind::Clang};
904+
if (hasDependency(depID)) {
787905
auto priorClangModuleDetails =
788-
findKnownDependency(dep.first).getAsClangModule();
789-
DEBUG_ASSERT(priorClangModuleDetails && newClangModuleDetails);
906+
findKnownDependency(depID).getAsClangModule();
907+
DEBUG_ASSERT(priorClangModuleDetails);
790908
auto priorContextHash = priorClangModuleDetails->contextHash;
791-
auto newContextHash = newClangModuleDetails->contextHash;
909+
auto newContextHash = dep.ID.ContextHash;
792910
if (priorContextHash != newContextHash) {
793911
// This situation means that within the same scanning action, Clang
794912
// Dependency Scanner has produced two different variants of the same
@@ -799,28 +917,33 @@ void ModuleDependenciesCache::recordClangDependencies(
799917
//
800918
// Emit a failure diagnostic here that is hopefully more actionable
801919
// for the time being.
802-
diags.diagnose(SourceLoc(), diag::dependency_scan_unexpected_variant,
803-
dep.first.ModuleName);
804-
diags.diagnose(
920+
ctx.Diags.diagnose(SourceLoc(),
921+
diag::dependency_scan_unexpected_variant,
922+
dep.ID.ModuleName);
923+
ctx.Diags.diagnose(
805924
SourceLoc(),
806925
diag::dependency_scan_unexpected_variant_context_hash_note,
807926
priorContextHash, newContextHash);
808-
diags.diagnose(
927+
ctx.Diags.diagnose(
809928
SourceLoc(),
810929
diag::dependency_scan_unexpected_variant_module_map_note,
811-
priorClangModuleDetails->moduleMapFile,
812-
newClangModuleDetails->moduleMapFile);
930+
priorClangModuleDetails->moduleMapFile, dep.ClangModuleMapFile);
931+
932+
auto newClangModuleDetails =
933+
dependencies::bridgeClangModuleDependency(
934+
ctx, dep, lookupModuleOutput, remapPath)
935+
.getAsClangModule();
813936

814937
auto diagnoseExtraCommandLineFlags =
815-
[&diags](const ClangModuleDependencyStorage *checkModuleDetails,
816-
const ClangModuleDependencyStorage *baseModuleDetails,
817-
bool isNewlyDiscovered) -> void {
938+
[&ctx](const ClangModuleDependencyStorage *checkModuleDetails,
939+
const ClangModuleDependencyStorage *baseModuleDetails,
940+
bool isNewlyDiscovered) -> void {
818941
std::unordered_set<std::string> baseCommandLineSet(
819942
baseModuleDetails->buildCommandLine.begin(),
820943
baseModuleDetails->buildCommandLine.end());
821944
for (const auto &checkArg : checkModuleDetails->buildCommandLine)
822945
if (baseCommandLineSet.find(checkArg) == baseCommandLineSet.end())
823-
diags.diagnose(
946+
ctx.Diags.diagnose(
824947
SourceLoc(),
825948
diag::dependency_scan_unexpected_variant_extra_arg_note,
826949
isNewlyDiscovered, checkArg);
@@ -831,9 +954,10 @@ void ModuleDependenciesCache::recordClangDependencies(
831954
priorClangModuleDetails, false);
832955
}
833956
} else {
834-
recordDependency(dep.first.ModuleName, dep.second);
835-
addSeenClangModule(clang::tooling::dependencies::ModuleID{
836-
dep.first.ModuleName, newClangModuleDetails->contextHash});
957+
recordDependency(dep.ID.ModuleName,
958+
dependencies::bridgeClangModuleDependency(
959+
ctx, dep, lookupModuleOutput, remapPath));
960+
addSeenClangModule(dep.ID);
837961
}
838962
}
839963
}

0 commit comments

Comments
 (0)