Skip to content

Commit f345188

Browse files
artemcmartemcm
committed
[Dependency Scanning] Bridge Clang dependency scanner results on-demand
Instead of always bridging all of the discovered modules of all of the queries, only do so for modules which are not already cached
1 parent 36246a2 commit f345188

File tree

7 files changed

+183
-185
lines changed

7 files changed

+183
-185
lines changed

‎include/swift/AST/ModuleDependencies.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@
2323
#include "swift/Basic/Assertions.h"
2424
#include "swift/Basic/CXXStdlibKind.h"
2525
#include "swift/Basic/LLVM.h"
26+
#include "swift/ClangImporter/ClangImporter.h"
2627
#include "swift/Serialization/Validation.h"
2728
#include "clang/CAS/CASOptions.h"
2829
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
30+
#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
2931
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
3032
#include "llvm/ADT/ArrayRef.h"
3133
#include "llvm/ADT/DenseSet.h"
@@ -1107,8 +1109,10 @@ class ModuleDependenciesCache {
11071109
ModuleDependencyInfo dependencies);
11081110

11091111
/// Record dependencies for the given collection of Clang modules.
1110-
void recordClangDependencies(ModuleDependencyVector moduleDependencies,
1111-
DiagnosticEngine &diags);
1112+
void recordClangDependencies(const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
1113+
const ASTContext &ctx,
1114+
clang::tooling::dependencies::LookupModuleOutputCallback LookupModuleOutput,
1115+
ClangImporter::RemapPathCallback remapPath = nullptr);
11121116

11131117
/// Update stored dependencies for the given module.
11141118
void updateDependency(ModuleDependencyID moduleID,

‎include/swift/ClangImporter/ClangImporter.h

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -497,11 +497,10 @@ class ClangImporter final : public ClangModuleLoader {
497497
llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
498498
clang::tooling::dependencies::ModuleOutputKind)>;
499499

500-
static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
501-
bridgeClangModuleDependencies(
500+
static ModuleDependencyInfo
501+
bridgeClangModuleDependency(
502502
const ASTContext &ctx,
503-
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
504-
clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
503+
const clang::tooling::dependencies::ModuleDeps &clangDependency,
505504
LookupModuleOutputCallback LookupModuleOutput,
506505
RemapPathCallback remapPath = nullptr);
507506

‎include/swift/DependencyScan/ModuleDependencyScanner.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ class ModuleDependencyScanningWorker {
4040

4141
private:
4242
/// Retrieve the module dependencies for the Clang module with the given name.
43-
ClangModuleScannerQueryResult scanFilesystemForClangModuleDependency(
43+
std::optional<clang::tooling::dependencies::TranslationUnitDeps>
44+
scanFilesystemForClangModuleDependency(
4445
Identifier moduleName,
4546
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
4647
&alreadySeenModules);

‎include/swift/Serialization/ScanningLoaders.h

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,6 @@ struct SwiftModuleScannerQueryResult {
4040
std::vector<IncompatibleCandidate> incompatibleCandidates;
4141
};
4242

43-
/// Result of looking up a Clang module on the current filesystem
44-
/// search paths.
45-
struct ClangModuleScannerQueryResult {
46-
ClangModuleScannerQueryResult(const ModuleDependencyVector &dependencyModuleGraph,
47-
const std::vector<std::string> &visibleModuleIdentifiers)
48-
: foundDependencyModuleGraph(dependencyModuleGraph),
49-
visibleModuleIdentifiers(visibleModuleIdentifiers) {}
50-
51-
ModuleDependencyVector foundDependencyModuleGraph;
52-
std::vector<std::string> visibleModuleIdentifiers;
53-
};
54-
5543
/// A module "loader" that looks for .swiftinterface and .swiftmodule files
5644
/// for the purpose of determining dependencies, but does not attempt to
5745
/// load the module files.

‎lib/AST/ModuleDependencies.cpp

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -779,16 +779,19 @@ void ModuleDependenciesCache::recordDependency(
779779
}
780780

781781
void ModuleDependenciesCache::recordClangDependencies(
782-
ModuleDependencyVector dependencies, DiagnosticEngine &diags) {
782+
const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
783+
const ASTContext &ctx,
784+
clang::tooling::dependencies::LookupModuleOutputCallback lookupModuleOutput,
785+
ClangImporter::RemapPathCallback remapPath) {
783786
for (const auto &dep : dependencies) {
784-
ASSERT(dep.first.Kind == ModuleDependencyKind::Clang);
785-
auto newClangModuleDetails = dep.second.getAsClangModule();
786-
if (hasDependency(dep.first)) {
787+
auto depID =
788+
ModuleDependencyID{dep.ID.ModuleName, ModuleDependencyKind::Clang};
789+
if (hasDependency(depID)) {
787790
auto priorClangModuleDetails =
788-
findKnownDependency(dep.first).getAsClangModule();
789-
DEBUG_ASSERT(priorClangModuleDetails && newClangModuleDetails);
791+
findKnownDependency(depID).getAsClangModule();
792+
DEBUG_ASSERT(priorClangModuleDetails);
790793
auto priorContextHash = priorClangModuleDetails->contextHash;
791-
auto newContextHash = newClangModuleDetails->contextHash;
794+
auto newContextHash = dep.ID.ContextHash;
792795
if (priorContextHash != newContextHash) {
793796
// This situation means that within the same scanning action, Clang
794797
// Dependency Scanner has produced two different variants of the same
@@ -799,28 +802,33 @@ void ModuleDependenciesCache::recordClangDependencies(
799802
//
800803
// Emit a failure diagnostic here that is hopefully more actionable
801804
// for the time being.
802-
diags.diagnose(SourceLoc(), diag::dependency_scan_unexpected_variant,
803-
dep.first.ModuleName);
804-
diags.diagnose(
805+
ctx.Diags.diagnose(SourceLoc(),
806+
diag::dependency_scan_unexpected_variant,
807+
dep.ID.ModuleName);
808+
ctx.Diags.diagnose(
805809
SourceLoc(),
806810
diag::dependency_scan_unexpected_variant_context_hash_note,
807811
priorContextHash, newContextHash);
808-
diags.diagnose(
812+
ctx.Diags.diagnose(
809813
SourceLoc(),
810814
diag::dependency_scan_unexpected_variant_module_map_note,
811-
priorClangModuleDetails->moduleMapFile,
812-
newClangModuleDetails->moduleMapFile);
815+
priorClangModuleDetails->moduleMapFile, dep.ClangModuleMapFile);
816+
817+
auto newClangModuleDetails =
818+
ClangImporter::bridgeClangModuleDependency(
819+
ctx, dep, lookupModuleOutput, remapPath)
820+
.getAsClangModule();
813821

814822
auto diagnoseExtraCommandLineFlags =
815-
[&diags](const ClangModuleDependencyStorage *checkModuleDetails,
816-
const ClangModuleDependencyStorage *baseModuleDetails,
817-
bool isNewlyDiscovered) -> void {
823+
[&ctx](const ClangModuleDependencyStorage *checkModuleDetails,
824+
const ClangModuleDependencyStorage *baseModuleDetails,
825+
bool isNewlyDiscovered) -> void {
818826
std::unordered_set<std::string> baseCommandLineSet(
819827
baseModuleDetails->buildCommandLine.begin(),
820828
baseModuleDetails->buildCommandLine.end());
821829
for (const auto &checkArg : checkModuleDetails->buildCommandLine)
822830
if (baseCommandLineSet.find(checkArg) == baseCommandLineSet.end())
823-
diags.diagnose(
831+
ctx.Diags.diagnose(
824832
SourceLoc(),
825833
diag::dependency_scan_unexpected_variant_extra_arg_note,
826834
isNewlyDiscovered, checkArg);
@@ -831,9 +839,10 @@ void ModuleDependenciesCache::recordClangDependencies(
831839
priorClangModuleDetails, false);
832840
}
833841
} else {
834-
recordDependency(dep.first.ModuleName, dep.second);
835-
addSeenClangModule(clang::tooling::dependencies::ModuleID{
836-
dep.first.ModuleName, newClangModuleDetails->contextHash});
842+
recordDependency(dep.ID.ModuleName,
843+
ClangImporter::bridgeClangModuleDependency(
844+
ctx, dep, lookupModuleOutput, remapPath));
845+
addSeenClangModule(dep.ID);
837846
}
838847
}
839848
}

‎lib/ClangImporter/ClangModuleDependencyScanner.cpp

Lines changed: 97 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -82,124 +82,117 @@ std::vector<std::string> ClangImporter::getClangDepScanningInvocationArguments(
8282
return commandLineArgs;
8383
}
8484

85-
ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
85+
ModuleDependencyInfo
86+
ClangImporter::bridgeClangModuleDependency(
8687
const ASTContext &ctx,
87-
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
88-
clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
89-
LookupModuleOutputCallback lookupModuleOutput,
90-
RemapPathCallback callback) {
91-
ModuleDependencyVector result;
92-
88+
const clang::tooling::dependencies::ModuleDeps &clangModuleDep,
89+
LookupModuleOutputCallback lookupModuleOutput, RemapPathCallback callback) {
9390
auto remapPath = [&](StringRef path) {
9491
if (callback)
9592
return callback(path);
9693
return path.str();
9794
};
9895

99-
for (auto &clangModuleDep : clangDependencies) {
100-
// File dependencies for this module.
101-
std::vector<std::string> fileDeps;
102-
clangModuleDep.forEachFileDep(
103-
[&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); });
104-
105-
std::vector<std::string> swiftArgs;
106-
auto addClangArg = [&](Twine arg) {
107-
swiftArgs.push_back("-Xcc");
108-
swiftArgs.push_back(arg.str());
109-
};
110-
111-
// We are using Swift frontend mode.
112-
swiftArgs.push_back("-frontend");
113-
114-
// Swift frontend action: -emit-pcm
115-
swiftArgs.push_back("-emit-pcm");
116-
swiftArgs.push_back("-module-name");
117-
swiftArgs.push_back(clangModuleDep.ID.ModuleName);
118-
119-
auto pcmPath = lookupModuleOutput(clangModuleDep,
120-
ModuleOutputKind::ModuleFile);
121-
swiftArgs.push_back("-o");
122-
swiftArgs.push_back(pcmPath);
123-
124-
// Ensure that the resulting PCM build invocation uses Clang frontend
125-
// directly
126-
swiftArgs.push_back("-direct-clang-cc1-module-build");
127-
128-
// Swift frontend option for input file path (Foo.modulemap).
129-
swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile));
130-
131-
auto invocation = clangModuleDep.getUnderlyingCompilerInvocation();
132-
// Clear some options from clang scanner.
133-
invocation.getMutFrontendOpts().ModuleCacheKeys.clear();
134-
invocation.getMutFrontendOpts().PathPrefixMappings.clear();
135-
invocation.getMutFrontendOpts().OutputFile.clear();
136-
137-
// Reset CASOptions since that should be coming from swift.
138-
invocation.getMutCASOpts() = clang::CASOptions();
139-
invocation.getMutFrontendOpts().CASIncludeTreeID.clear();
140-
141-
// FIXME: workaround for rdar://105684525: find the -ivfsoverlay option
142-
// from clang scanner and pass to swift.
143-
if (!ctx.CASOpts.EnableCaching) {
144-
auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles;
145-
for (auto overlay : overlayFiles) {
146-
swiftArgs.push_back("-vfsoverlay");
147-
swiftArgs.push_back(overlay);
148-
}
149-
}
96+
// File dependencies for this module.
97+
std::vector<std::string> fileDeps;
98+
clangModuleDep.forEachFileDep(
99+
[&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); });
150100

151-
// Add args reported by the scanner.
152-
auto clangArgs = invocation.getCC1CommandLine();
153-
llvm::for_each(clangArgs, addClangArg);
101+
std::vector<std::string> swiftArgs;
102+
auto addClangArg = [&](Twine arg) {
103+
swiftArgs.push_back("-Xcc");
104+
swiftArgs.push_back(arg.str());
105+
};
154106

155-
// CASFileSystemRootID.
156-
std::string RootID = clangModuleDep.CASFileSystemRootID
157-
? clangModuleDep.CASFileSystemRootID->toString()
158-
: "";
107+
// We are using Swift frontend mode.
108+
swiftArgs.push_back("-frontend");
159109

160-
std::string IncludeTree =
161-
clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
110+
// Swift frontend action: -emit-pcm
111+
swiftArgs.push_back("-emit-pcm");
112+
swiftArgs.push_back("-module-name");
113+
swiftArgs.push_back(clangModuleDep.ID.ModuleName);
162114

163-
ctx.CASOpts.enumerateCASConfigurationFlags(
164-
[&](StringRef Arg) { swiftArgs.push_back(Arg.str()); });
115+
auto pcmPath =
116+
lookupModuleOutput(clangModuleDep, ModuleOutputKind::ModuleFile);
117+
swiftArgs.push_back("-o");
118+
swiftArgs.push_back(pcmPath);
165119

166-
if (!IncludeTree.empty()) {
167-
swiftArgs.push_back("-clang-include-tree-root");
168-
swiftArgs.push_back(IncludeTree);
169-
}
170-
std::string mappedPCMPath = remapPath(pcmPath);
171-
172-
std::vector<LinkLibrary> LinkLibraries;
173-
for (const auto &ll : clangModuleDep.LinkLibraries)
174-
LinkLibraries.emplace_back(
175-
ll.Library,
176-
ll.IsFramework ? LibraryKind::Framework : LibraryKind::Library,
177-
/*static=*/false);
178-
179-
// Module-level dependencies.
180-
llvm::StringSet<> alreadyAddedModules;
181-
auto dependencies = ModuleDependencyInfo::forClangModule(
182-
pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
183-
clangModuleDep.ID.ContextHash, swiftArgs, fileDeps,
184-
LinkLibraries, RootID, IncludeTree, /*module-cache-key*/ "",
185-
clangModuleDep.IsSystem);
186-
187-
std::vector<ModuleDependencyID> directDependencyIDs;
188-
for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
189-
// FIXME: This assumes, conservatively, that all Clang module imports
190-
// are exported. We need to fix this once the clang scanner gains the appropriate
191-
// API to query this.
192-
dependencies.addModuleImport(moduleName.ModuleName, /* isExported */ true,
193-
AccessLevel::Public, &alreadyAddedModules);
194-
// It is safe to assume that all dependencies of a Clang module are Clang modules.
195-
directDependencyIDs.push_back({moduleName.ModuleName, ModuleDependencyKind::Clang});
120+
// Ensure that the resulting PCM build invocation uses Clang frontend
121+
// directly
122+
swiftArgs.push_back("-direct-clang-cc1-module-build");
123+
124+
// Swift frontend option for input file path (Foo.modulemap).
125+
swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile));
126+
127+
auto invocation = clangModuleDep.getUnderlyingCompilerInvocation();
128+
// Clear some options from clang scanner.
129+
invocation.getMutFrontendOpts().ModuleCacheKeys.clear();
130+
invocation.getMutFrontendOpts().PathPrefixMappings.clear();
131+
invocation.getMutFrontendOpts().OutputFile.clear();
132+
133+
// Reset CASOptions since that should be coming from swift.
134+
invocation.getMutCASOpts() = clang::CASOptions();
135+
invocation.getMutFrontendOpts().CASIncludeTreeID.clear();
136+
137+
// FIXME: workaround for rdar://105684525: find the -ivfsoverlay option
138+
// from clang scanner and pass to swift.
139+
if (!ctx.CASOpts.EnableCaching) {
140+
auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles;
141+
for (auto overlay : overlayFiles) {
142+
swiftArgs.push_back("-vfsoverlay");
143+
swiftArgs.push_back(overlay);
196144
}
197-
dependencies.setImportedClangDependencies(directDependencyIDs);
198-
result.push_back(std::make_pair(ModuleDependencyID{clangModuleDep.ID.ModuleName,
199-
ModuleDependencyKind::Clang},
200-
dependencies));
201145
}
202-
return result;
146+
147+
// Add args reported by the scanner.
148+
auto clangArgs = invocation.getCC1CommandLine();
149+
llvm::for_each(clangArgs, addClangArg);
150+
151+
// CASFileSystemRootID.
152+
std::string RootID = clangModuleDep.CASFileSystemRootID
153+
? clangModuleDep.CASFileSystemRootID->toString()
154+
: "";
155+
156+
std::string IncludeTree =
157+
clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
158+
159+
ctx.CASOpts.enumerateCASConfigurationFlags(
160+
[&](StringRef Arg) { swiftArgs.push_back(Arg.str()); });
161+
162+
if (!IncludeTree.empty()) {
163+
swiftArgs.push_back("-clang-include-tree-root");
164+
swiftArgs.push_back(IncludeTree);
165+
}
166+
std::string mappedPCMPath = remapPath(pcmPath);
167+
168+
std::vector<LinkLibrary> LinkLibraries;
169+
for (const auto &ll : clangModuleDep.LinkLibraries)
170+
LinkLibraries.emplace_back(ll.Library,
171+
ll.IsFramework ? LibraryKind::Framework
172+
: LibraryKind::Library,
173+
/*static=*/false);
174+
175+
// Module-level dependencies.
176+
llvm::StringSet<> alreadyAddedModules;
177+
auto bridgedDependencyInfo = ModuleDependencyInfo::forClangModule(
178+
pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
179+
clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, LinkLibraries, RootID,
180+
IncludeTree, /*module-cache-key*/ "", clangModuleDep.IsSystem);
181+
182+
std::vector<ModuleDependencyID> directDependencyIDs;
183+
for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
184+
// FIXME: This assumes, conservatively, that all Clang module imports
185+
// are exported. We need to fix this once the clang scanner gains the
186+
// appropriate API to query this.
187+
bridgedDependencyInfo.addModuleImport(moduleName.ModuleName, /* isExported */ true,
188+
AccessLevel::Public, &alreadyAddedModules);
189+
// It is safe to assume that all dependencies of a Clang module are Clang
190+
// modules.
191+
directDependencyIDs.push_back(
192+
{moduleName.ModuleName, ModuleDependencyKind::Clang});
193+
}
194+
bridgedDependencyInfo.setImportedClangDependencies(directDependencyIDs);
195+
return bridgedDependencyInfo;
203196
}
204197

205198
void ClangImporter::getBridgingHeaderOptions(

0 commit comments

Comments
 (0)