Skip to content

Commit ad1e06a

Browse files
owenvowenv
authored
Adopt libclang working directory optimization API (#361)
Adopt new libclang API which the scanner uses to notify the build system when it is ok to compile a module using an arbitrary current working directory to facilitate increased module sharing. The relevant API was introduced in swiftlang/llvm-project#10146. rdar://145991369
1 parent 7ff90c9 commit ad1e06a

File tree

8 files changed

+159
-6
lines changed

8 files changed

+159
-6
lines changed

‎Sources/SWBCSupport/CLibclang.cpp‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,18 @@ extern "C" {
892892
*/
893893
void (*clang_experimental_DependencyScannerServiceOptions_setCASOptions)(CXDependencyScannerServiceOptions Opts, CXCASOptions);
894894

895+
/**
896+
* Set the working directory optimization option.
897+
* The dependency scanner service option Opts will indicate to the scanner that
898+
* the current working directory can or cannot be ignored when computing the
899+
* pcms' context hashes. The scanner will then determine if it is safe to
900+
* optimize each module and act accordingly.
901+
*
902+
* \param Value If it is non zero, the option is on. Otherwise the
903+
* option is off.
904+
*/
905+
void (*clang_experimental_DependencyScannerServiceOptions_setCWDOptimization)(CXDependencyScannerServiceOptions Opts, int Value);
906+
895907
/**
896908
* Create a \c CXDependencyScannerService object.
897909
* Must be disposed with \c clang_experimental_DependencyScannerService_dispose_v0().
@@ -1141,6 +1153,12 @@ extern "C" {
11411153
*/
11421154
const char *(*clang_experimental_DepGraphModule_getCacheKey)(CXDepGraphModule);
11431155

1156+
/**
1157+
* \returns 1 if the scanner ignores the current working directory when
1158+
* computing the module's context hash. Otherwise returns 0.
1159+
*/
1160+
int (*clang_experimental_DepGraphModule_isCWDIgnored)(CXDepGraphModule);
1161+
11441162
/**
11451163
* \returns the number \c CXDepGraphTUCommand objects in the graph.
11461164
*/
@@ -1410,6 +1428,7 @@ struct LibclangWrapper {
14101428
LOOKUP_OPTIONAL(clang_experimental_DependencyScannerServiceOptions_setDependencyMode);
14111429
LOOKUP_OPTIONAL_CAS_API(clang_experimental_DependencyScannerServiceOptions_setCASDatabases);
14121430
LOOKUP_OPTIONAL(clang_experimental_DependencyScannerServiceOptions_setCASOptions);
1431+
LOOKUP_OPTIONAL(clang_experimental_DependencyScannerServiceOptions_setCWDOptimization);
14131432
LOOKUP_OPTIONAL(clang_experimental_DependencyScannerService_create_v1);
14141433
LOOKUP_OPTIONAL_DEPENDENCY_SCANNER_API(clang_experimental_DependencyScannerService_dispose_v0);
14151434
LOOKUP_OPTIONAL_DEPENDENCY_SCANNER_API(clang_experimental_DependencyScannerWorker_create_v0);
@@ -1429,6 +1448,7 @@ struct LibclangWrapper {
14291448
LOOKUP_OPTIONAL(clang_experimental_DepGraphModule_getModuleDeps);
14301449
LOOKUP_OPTIONAL(clang_experimental_DepGraphModule_getBuildArguments);
14311450
LOOKUP_OPTIONAL(clang_experimental_DepGraphModule_getCacheKey);
1451+
LOOKUP_OPTIONAL(clang_experimental_DepGraphModule_isCWDIgnored);
14321452
LOOKUP_OPTIONAL(clang_experimental_DepGraphModule_getIncludeTreeID);
14331453
LOOKUP_OPTIONAL(clang_experimental_DepGraph_getNumTUCommands);
14341454
LOOKUP_OPTIONAL(clang_experimental_DepGraph_getTUCommand);
@@ -1574,6 +1594,9 @@ struct LibclangScanner {
15741594
if (casOpts) {
15751595
lib->fns.clang_experimental_DependencyScannerServiceOptions_setCASOptions(opts, casOpts->casOpts);
15761596
}
1597+
if (lib->fns.clang_experimental_DependencyScannerServiceOptions_setCWDOptimization) {
1598+
lib->fns.clang_experimental_DependencyScannerServiceOptions_setCWDOptimization(opts, 1);
1599+
}
15771600
service = lib->fns.clang_experimental_DependencyScannerService_create_v1(opts);
15781601
assert(service && "unable to create service");
15791602
lib->fns.clang_experimental_DependencyScannerServiceOptions_dispose(opts);
@@ -1747,6 +1770,11 @@ extern "C" {
17471770
lib->wrapper->fns.clang_experimental_cas_isMaterialized;
17481771
}
17491772

1773+
bool libclang_has_current_working_directory_optimization(libclang_t lib) {
1774+
return lib->wrapper->fns.clang_experimental_DepGraphModule_isCWDIgnored &&
1775+
lib->wrapper->fns.clang_experimental_DependencyScannerServiceOptions_setCWDOptimization;
1776+
}
1777+
17501778
libclang_casoptions_t libclang_casoptions_create(libclang_t lib) {
17511779
auto opts = lib->wrapper->fns.clang_experimental_cas_Options_create();
17521780
return new libclang_casoptions_t_{{lib->wrapper, opts}};
@@ -2055,6 +2083,7 @@ extern "C" {
20552083
const char *includeTreeID = lib->fns.clang_experimental_DepGraphModule_getIncludeTreeID
20562084
? lib->fns.clang_experimental_DepGraphModule_getIncludeTreeID(depMod)
20572085
: nullptr;
2086+
bool isCWDIgnored = lib->fns.clang_experimental_DepGraphModule_isCWDIgnored ? lib->fns.clang_experimental_DepGraphModule_isCWDIgnored(depMod) : 0;
20582087
LibclangFunctions::CXCStringArray fileDeps = lib->fns.clang_experimental_DepGraphModule_getFileDeps(depMod);
20592088
LibclangFunctions::CXCStringArray moduleDeps = lib->fns.clang_experimental_DepGraphModule_getModuleDeps(depMod);
20602089
LibclangFunctions::CXCStringArray buildArguments = lib->fns.clang_experimental_DepGraphModule_getBuildArguments(depMod);
@@ -2063,6 +2092,7 @@ extern "C" {
20632092
modules[i].context_hash = contextHash;
20642093
modules[i].module_map_path = moduleMapPath;
20652094
modules[i].include_tree_id = includeTreeID;
2095+
modules[i].is_cwd_ignored = isCWDIgnored;
20662096
modules[i].cache_key = cacheKey;
20672097
modules[i].file_deps = copyStringSet(fileDeps);
20682098
modules[i].module_deps = copyStringSet(moduleDeps);

‎Sources/SWBCSupport/CLibclang.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ typedef struct {
4949
const char *module_map_path;
5050
const char **file_deps;
5151
const char *include_tree_id;
52+
bool is_cwd_ignored;
5253
const char **module_deps;
5354
const char *cache_key;
5455
const char **build_arguments;
@@ -122,6 +123,9 @@ bool libclang_has_cas_pruning_feature(libclang_t lib);
122123
/// Whether the libclang has CAS up-to-date checking support.
123124
bool libclang_has_cas_up_to_date_checks_feature(libclang_t lib);
124125

126+
/// Whether the libclang has current working directory optimization support.
127+
bool libclang_has_current_working_directory_optimization(libclang_t lib);
128+
125129
/// Create the CAS options object.
126130
libclang_casoptions_t libclang_casoptions_create(libclang_t lib);
127131

‎Sources/SWBCore/LibclangVendored/Libclang.swift‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ public final class Libclang {
9898
public var supportsCASUpToDateChecks: Bool {
9999
libclang_has_cas_up_to_date_checks_feature(lib)
100100
}
101+
102+
public var supportsCurrentWorkingDirectoryOptimization: Bool {
103+
libclang_has_current_working_directory_optimization(lib)
104+
}
101105
}
102106

103107
enum DependencyScanningError: Error {
@@ -130,6 +134,7 @@ public final class DependencyScanner {
130134
public var module_deps: some Sequence<String> { clang_module_dependency.module_deps.toLazyStringSequence() }
131135
public var cache_key: String? { clang_module_dependency.cache_key.map { String(cString: $0) } }
132136
public var build_arguments: some Sequence<String> { clang_module_dependency.build_arguments.toLazyStringSequence() }
137+
public var is_cwd_ignored: Bool { clang_module_dependency.is_cwd_ignored }
133138
}
134139

135140
public struct Command {

‎Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -429,8 +429,8 @@ package final class ClangModuleDependencyGraph {
429429
fileDependencies: fileDependencies,
430430
includeTreeID: module.include_tree_id,
431431
moduleDependencies: OrderedSet(moduleDeps),
432-
// Cached builds do not rely on the process working directory, and different scanner working directories should not inhibit task deduplication
433-
workingDirectory: module.cache_key != nil ? Path.root : workingDirectory,
432+
// Cached builds do not rely on the process working directory, and different scanner working directories should not inhibit task deduplication. The same is true if the scanner reports the working directory can be ignored.
433+
workingDirectory: module.cache_key != nil || module.is_cwd_ignored ? Path.root : workingDirectory,
434434
command: DependencyInfo.CompileCommand(cacheKey: module.cache_key, arguments: commandLine),
435435
transitiveIncludeTreeIDs: transitiveIncludeTreeIDs,
436436
transitiveCompileCommandCacheKeys: transitiveCommandCacheKeys,

‎Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
144144
taskKey: .precompileClangModule(precompileModuleTaskKey),
145145
taskID: state.dynamicTaskBaseID,
146146
singleUse: true,
147-
workingDirectory: dependencyInfo.workingDirectory,
147+
workingDirectory: Path.root,
148148
environment: task.environment,
149149
forTarget: nil,
150150
priority: .preferred,

‎Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ final public class PrecompileClangModuleTaskAction: TaskAction, BuildValueValida
101101
taskKey: .precompileClangModule(taskKey),
102102
taskID: taskID,
103103
singleUse: false,
104-
workingDirectory: dependencyInfo.workingDirectory,
104+
workingDirectory: Path.root,
105105
environment: task.environment,
106106
forTarget: task.forTarget,
107107
priority: .preferred,
@@ -188,7 +188,7 @@ final public class PrecompileClangModuleTaskAction: TaskAction, BuildValueValida
188188
if try ClangCompileTaskAction.replayCachedCommand(
189189
command,
190190
casDBs: casDBs,
191-
workingDirectory: task.workingDirectory,
191+
workingDirectory: dependencyInfo.workingDirectory,
192192
outputDelegate: outputDelegate,
193193
enableDiagnosticRemarks: key.casOptions!.enableDiagnosticRemarks
194194
) {
@@ -198,7 +198,7 @@ final public class PrecompileClangModuleTaskAction: TaskAction, BuildValueValida
198198

199199
let delegate = TaskProcessDelegate(outputDelegate: outputDelegate)
200200
// The frontend invocations should be unaffected by the environment, pass an empty one.
201-
try await spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate)
201+
try await spawn(commandLine: commandLine, environment: [:], workingDirectory: dependencyInfo.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate)
202202

203203
let result = delegate.commandResult ?? .failed
204204
if result == .succeeded {

‎Sources/SWBTestSupport/SkippedTestSupport.swift‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,13 @@ extension Trait where Self == Testing.ConditionTrait {
346346
}
347347
}
348348

349+
package static var requireDependencyScannerWorkingDirectoryOptimization: Self {
350+
enabled {
351+
let libclang = try #require(try await ConditionTraitContext.shared.libclang)
352+
return libclang.supportsCurrentWorkingDirectoryOptimization
353+
}
354+
}
355+
349356
package static var requireCompilationCaching: Self {
350357
enabled("compilation caching is not supported") {
351358
try await ConditionTraitContext.shared.supportsCompilationCaching

‎Tests/SWBBuildSystemTests/ClangExplicitModulesTests.swift‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1374,6 +1374,113 @@ fileprivate struct ClangExplicitModulesTests: CoreBasedTests {
13741374
}
13751375
}
13761376

1377+
@Test(.requireSDKs(.macOS), .requireDependencyScanner, .requireDependencyScannerWorkingDirectoryOptimization)
1378+
func sharingBetweenProjects() async throws {
1379+
try await withTemporaryDirectory { tmpDir in
1380+
let testWorkspace = TestWorkspace(
1381+
"Test",
1382+
sourceRoot: tmpDir.join("Test"),
1383+
projects: [
1384+
TestProject(
1385+
"aProject",
1386+
groupTree: TestGroup(
1387+
"Sources",
1388+
children: [
1389+
TestFile("mod_nested.h"),
1390+
TestFile("mod.h"),
1391+
TestFile("module.modulemap"),
1392+
TestFile("file_1.c"),
1393+
]),
1394+
buildConfigurations: [TestBuildConfiguration(
1395+
"Debug",
1396+
buildSettings: [
1397+
"PRODUCT_NAME": "$(TARGET_NAME)",
1398+
"CLANG_ENABLE_MODULES": "YES",
1399+
"_EXPERIMENTAL_CLANG_EXPLICIT_MODULES": "YES",
1400+
"CLANG_EXPLICIT_MODULES_OUTPUT_PATH": "\(tmpDir.join("clangmodules").str)",
1401+
])],
1402+
targets: [
1403+
TestStandardTarget(
1404+
"Library_1",
1405+
type: .staticLibrary,
1406+
buildPhases: [
1407+
TestSourcesBuildPhase(["file_1.c"]),
1408+
]),
1409+
]), TestProject(
1410+
"aProject2",
1411+
groupTree: TestGroup(
1412+
"Sources",
1413+
children: [
1414+
TestFile("file_2.c"),
1415+
]),
1416+
buildConfigurations: [TestBuildConfiguration(
1417+
"Debug",
1418+
buildSettings: [
1419+
"PRODUCT_NAME": "$(TARGET_NAME)",
1420+
"CLANG_ENABLE_MODULES": "YES",
1421+
"_EXPERIMENTAL_CLANG_EXPLICIT_MODULES": "YES",
1422+
"HEADER_SEARCH_PATHS": "$(inherited) \(tmpDir.join("Test/aProject").str)",
1423+
"CLANG_EXPLICIT_MODULES_OUTPUT_PATH": "\(tmpDir.join("clangmodules").str)",
1424+
])],
1425+
targets: [
1426+
TestStandardTarget(
1427+
"Library_2",
1428+
type: .staticLibrary,
1429+
buildPhases: [
1430+
TestSourcesBuildPhase(["file_2.c"]),
1431+
]),
1432+
])])
1433+
1434+
let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false)
1435+
1436+
try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/module.modulemap")) { stream in
1437+
stream <<<
1438+
"""
1439+
module mod { header "mod.h" }
1440+
"""
1441+
}
1442+
1443+
try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/mod.h")) { stream in
1444+
stream <<<
1445+
"""
1446+
void foo(void) {}
1447+
"""
1448+
}
1449+
1450+
try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file_1.c")) { stream in
1451+
stream <<<
1452+
"""
1453+
#include "mod.h"
1454+
"""
1455+
}
1456+
1457+
try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject2/file_2.c")) { stream in
1458+
stream <<<
1459+
"""
1460+
#include "mod.h"
1461+
"""
1462+
}
1463+
1464+
let parameters = BuildParameters(configuration: "Debug")
1465+
let buildRequest = BuildRequest(parameters: parameters, buildTargets: tester.workspace.projects[0].targets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }) + tester.workspace.projects[1].targets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }), continueBuildingAfterErrors: false, useParallelTargets: true, useImplicitDependencies: false, useDryRun: false)
1466+
1467+
try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
1468+
1469+
for targetName in ["Library_1", "Library_2"] {
1470+
results.checkTaskExists(.matchTargetName(targetName), .matchRuleType("ScanDependencies"))
1471+
results.checkTaskExists(.matchTargetName(targetName), .matchRuleType("CompileC"))
1472+
}
1473+
1474+
// We should optimize out the working directory and only build one variant of "mod"
1475+
results.checkTasks(.matchRulePattern(["PrecompileModule", .contains("mod-")])) { tasks in
1476+
#expect(tasks.count == 1)
1477+
}
1478+
1479+
results.checkNoDiagnostics()
1480+
}
1481+
}
1482+
}
1483+
13771484
@Test(.requireSDKs(.macOS))
13781485
func incrementalBuildBasics() async throws {
13791486
try await withTemporaryDirectory { tmpDir in

0 commit comments

Comments
 (0)