Skip to content

Take the host platform into account when writing clang response files #810

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Take the host platform into account when writing clang response files #810

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Sources/SWBCore/Settings/BuiltinMacros.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1189,6 +1189,7 @@ public final class BuiltinMacros {
public static let _WRAPPER_PARENT_PATH = BuiltinMacros.declareStringMacro("_WRAPPER_PARENT_PATH")
public static let _WRAPPER_RESOURCES_DIR = BuiltinMacros.declareStringMacro("_WRAPPER_RESOURCES_DIR")
public static let __INPUT_FILE_LIST_PATH__ = BuiltinMacros.declarePathMacro("__INPUT_FILE_LIST_PATH__")
public static let HOST_RESPONSE_FILE_FORMAT = BuiltinMacros.declareEnumMacro("HOST_RESPONSE_FILE_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
public static let LINKER_FILE_LIST_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_FILE_LIST_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
public static let LINKER_RESPONSE_FILE_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_RESPONSE_FILE_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
public static let SWIFT_RESPONSE_FILE_PATH = BuiltinMacros.declarePathMacro("SWIFT_RESPONSE_FILE_PATH")
Expand Down Expand Up @@ -2443,6 +2444,7 @@ public final class BuiltinMacros {
_WRAPPER_PARENT_PATH,
_WRAPPER_RESOURCES_DIR,
__INPUT_FILE_LIST_PATH__,
HOST_RESPONSE_FILE_FORMAT,
LINKER_FILE_LIST_FORMAT,
LINKER_RESPONSE_FILE_FORMAT,
__ARCHS__,
Expand Down
2 changes: 2 additions & 0 deletions Sources/SWBCore/Settings/Settings.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2350,6 +2350,8 @@ private class SettingsBuilder {
platformTable.push(BuiltinMacros.NATIVE_ARCH, literal: Architecture.hostStringValue ?? fallbackArch)
}

platformTable.push(BuiltinMacros.HOST_RESPONSE_FILE_FORMAT, literal: workspaceContext.core.hostOperatingSystem.defaultResponseFileFormat)

// Add the platform deployment target defaults, for real platforms.
//
// FIXME: For legacy compatibility, we only do this when configuring settings for a target. This distinction most likely isn't important, and should just be eliminated.
Expand Down
22 changes: 14 additions & 8 deletions Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,11 @@ public struct ClangPrefixInfo: Serializable, Hashable, Encodable, Sendable {
}
}

public struct ClangResponseFileInfo: SerializableCodable, Sendable {
var attachmentPath: Path
var format: ResponseFileFormat
}

/// The minimal data we need to serialize to reconstruct `ClangSourceFileIndexingInfo` from `generateIndexingInfo`
public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
let sourceFileIndex: Int
Expand All @@ -102,7 +107,7 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
let workingDir: Path
let prefixInfo: ClangPrefixInfo?
public let toolchains: [String]
let responseFileAttachmentPaths: [Path: Path]
let responseFileAttachmentPaths: [Path: ClangResponseFileInfo]

init(sourceFileIndex: Int,
outputFileIndex: Int,
Expand All @@ -112,7 +117,7 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
workingDir: Path,
prefixInfo: ClangPrefixInfo?,
toolchains: [String],
responseFileAttachmentPaths: [Path: Path]) {
responseFileAttachmentPaths: [Path: ClangResponseFileInfo]) {
self.sourceFileIndex = sourceFileIndex
self.outputFileIndex = outputFileIndex
self.sourceLanguageIndex = sourceLanguageIndex
Expand Down Expand Up @@ -202,7 +207,7 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
static let skippedArgsWithoutValues = Set<ByteString>(["-M", "-MD", "-MMD", "-MG", "-MJ", "-MM", "-MP", "-MV", "-fmodules-validate-once-per-build-session"])
static let skippedArgsWithValues = Set<ByteString>(["-MT", "-MF", "-MQ", "--serialize-diagnostics"])

public static func indexingCommandLine(from commandLine: [ByteString], workingDir: Path, prefixInfo: ClangPrefixInfo? = nil, addSupplementary: Bool = true, replaceCompile: Bool = true, responseFileMapping: [Path: Path]) -> [ByteString] {
public static func indexingCommandLine(from commandLine: [ByteString], workingDir: Path, prefixInfo: ClangPrefixInfo? = nil, addSupplementary: Bool = true, replaceCompile: Bool = true, responseFileMapping: [Path: ClangResponseFileInfo]) -> [ByteString] {
var result = [ByteString]()
var iterator = commandLine.makeIterator()
let _ = iterator.next() // Skip compiler path
Expand Down Expand Up @@ -234,8 +239,8 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
} else if arg.bytes.starts(with: ByteString(stringLiteral: "-fbuild-session-file=").bytes) {
// Skip
} else if arg.starts(with: ByteString(unicodeScalarLiteral: "@")),
let attachmentPath = responseFileMapping[Path(arg.asString.dropFirst())],
let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: .unixShellQuotedSpaceSeparated) {
let attachment = responseFileMapping[Path(arg.asString.dropFirst())],
let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachment.attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: attachment.format) {
result.append(contentsOf: responseFileArgs.map { ByteString(encodingAsUTF8: $0) })
} else {
result.append(arg)
Expand Down Expand Up @@ -568,7 +573,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
let inputs: [Path]

/// Maps response files in `flags` to the corresponding recorded attachment in the build description.
let responseFileMapping: [Path: Path]
let responseFileMapping: [Path: ClangResponseFileInfo]

}

Expand Down Expand Up @@ -719,9 +724,10 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
ctx.add(string: self.identifier)

let responseFilePath = scope.evaluate(BuiltinMacros.PER_ARCH_OBJECT_FILE_DIR).join("\(ctx.signature.asString)-common-args.resp")
let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: .unixShellQuotedSpaceSeparated)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
let responseFileFormat = scope.evaluate(BuiltinMacros.HOST_RESPONSE_FILE_FORMAT)
let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: responseFileFormat)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])

return ConstantFlags(flags: regularCommandLine + ["@\(responseFilePath.str)"], headerSearchPaths: headerSearchPaths, inputs: [responseFilePath], responseFileMapping: [responseFilePath: attachmentPath])
return ConstantFlags(flags: regularCommandLine + ["@\(responseFilePath.str)"], headerSearchPaths: headerSearchPaths, inputs: [responseFilePath], responseFileMapping: [responseFilePath: .init(attachmentPath: attachmentPath, format: responseFileFormat)])
} else {
return ConstantFlags(flags: commandLine, headerSearchPaths: headerSearchPaths, inputs: [], responseFileMapping: [:])
}
Expand Down
9 changes: 9 additions & 0 deletions Sources/SWBUtil/ProcessInfo.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,15 @@ public enum OperatingSystem: Hashable, Sendable {
return .elf
}
}

public var defaultResponseFileFormat: ResponseFileFormat {
switch self {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: use this?

self == .windows ? .windowsShellQuotedNewlineSeparated : .unixShellQuotedSpaceSeparated

Seems unlikely there will ever be difference except for Windows

case .windows:
.windowsShellQuotedNewlineSeparated
case .macOS, .iOS, .tvOS, .watchOS, .visionOS, .linux, .freebsd, .openbsd, .android, .unknown:
.unixShellQuotedSpaceSeparated
}
}
}

public enum ImageFormat {
Expand Down
34 changes: 20 additions & 14 deletions Tests/SWBTaskConstructionTests/ClangResponseFileTaskConstructionTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,13 @@ fileprivate struct ClangResponseFileTaskConstructionTests: CoreBasedTests {
let responseFilePath = try #require(task.outputs.only)

// The command arguments in the response file vary vastly between different platforms, so just check for some basics present in the content.
let contentAsString = contents.asString
#expect(contentAsString.contains("-target "))
#expect(contentAsString.contains("Test-generated-files.hmap"))
#expect(contentAsString.contains("Test-own-target-headers.hmap"))
#expect(contentAsString.contains("Test-all-target-headers.hmap"))
if tester.core.hostOperatingSystem.defaultResponseFileFormat == .unixShellQuotedSpaceSeparated {
let contentAsString = contents.asString
#expect(contentAsString.contains("-target "))
#expect(contentAsString.contains("Test-generated-files.hmap"))
#expect(contentAsString.contains("Test-own-target-headers.hmap"))
#expect(contentAsString.contains("Test-all-target-headers.hmap"))
}

for name in ["a.c", "b.c", "c.c"] {
results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix(name))) { compileTask in
Expand Down Expand Up @@ -166,20 +168,22 @@ fileprivate struct ClangResponseFileTaskConstructionTests: CoreBasedTests {
let tester = try TaskConstructionTester(core, testProject)
await tester.checkBuild(runDestination: .host) { results in
results.checkWriteAuxiliaryFileTask(.matchRuleItemPattern(.suffix("-common-args.resp"))) { task, contents in
let stringContents = contents.asString
#expect(stringContents.contains("-target"))
let blocksFlag = switch runDestination {
if tester.core.hostOperatingSystem.defaultResponseFileFormat == .unixShellQuotedSpaceSeparated {
let stringContents = contents.asString
#expect(stringContents.contains("-target"))
let blocksFlag = switch runDestination {
case .macOS:
"-fasm-blocks"
case .linux:
"-fblocks"
default:
" "
}
#expect(stringContents.contains(blocksFlag))
#expect(!stringContents.contains("-MMD"))
#expect(!stringContents.contains("-fcolor-diagnostics"))
#expect(!stringContents.contains("-Wno-private-module"))
}
#expect(stringContents.contains(blocksFlag))
#expect(!stringContents.contains("-MMD"))
#expect(!stringContents.contains("-fcolor-diagnostics"))
#expect(!stringContents.contains("-Wno-private-module"))
for name in ["a", "b", "c"] {
results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix(name + ".c"))) { compileTask in
compileTask.checkCommandLineMatches(["-MMD", "-MT", "dependencies", "-MF", .suffix(name + ".d")])
Expand Down Expand Up @@ -349,8 +353,10 @@ fileprivate struct ClangResponseFileTaskConstructionTests: CoreBasedTests {
let tester = try TaskConstructionTester(core, testProject)
await tester.checkBuild(runDestination: .host) { results in
results.checkWriteAuxiliaryFileTask(.matchRuleItemPattern(.suffix("-common-args.resp"))) { task, contents in
let stringContents = contents.asString
#expect(stringContents.contains("-Xclang -Wno-shorten-64-to-32"))
if tester.core.hostOperatingSystem.defaultResponseFileFormat == .unixShellQuotedSpaceSeparated {
let stringContents = contents.asString
#expect(stringContents.contains("-Xclang -Wno-shorten-64-to-32"))
}
results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix("a.c"))) { compileTask in
compileTask.checkCommandLineDoesNotContain("-Xclang")
}
Expand Down
Loading