Skip to content

Commit d89a86f

Browse files
AlexDenisovAlexDenisov
authored
Merge pull request github#13979 from github/alexdenisov/autobuilder-spm
Swift: teach autobuilder about SPM, CocoaPods, and Carthage
2 parents 96e9dfc + 5cce37b commit d89a86f

File tree

12 files changed

+159
-79
lines changed

12 files changed

+159
-79
lines changed

‎swift/integration-tests/osx-only/autobuilder/failure/diagnostics.expected

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"markdownMessage": "`autobuild` failed to run the detected build command:\n\n```\n/usr/bin/xcodebuild build -project <test-root-directory>/hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
2+
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/xcodebuild build -project <test-root-directory>/hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
33
"severity": "error",
44
"source": {
55
"extractorName": "swift",

‎swift/integration-tests/osx-only/autobuilder/no-build-system/diagnostics.expected

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"markdownMessage": "`autobuild` could not detect an Xcode project or workspace.\n\nSet up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
2+
"markdownMessage": "`autobuild` detected neither an Xcode project or workspace, nor a Swift package\n\nSet up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
33
"severity": "error",
44
"source": {
55
"extractorName": "swift",

‎swift/integration-tests/osx-only/autobuilder/no-swift-with-spm/diagnostics.expected

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"markdownMessage": "A Swift package was detected, but no viable Xcode target was found.\n\nSwift Package Manager builds are not currently supported by `autobuild`. Set up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
2+
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/swift build --package-path <test-root-directory>/hello-objective\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
33
"severity": "error",
44
"source": {
55
"extractorName": "swift",
6-
"id": "swift/autobuilder/spm-not-supported",
7-
"name": "Swift Package Manager is not supported"
6+
"id": "swift/autobuilder/build-command-failed",
7+
"name": "Detected build command failed"
88
},
99
"visibility": {
1010
"cliSummaryTable": true,

‎swift/integration-tests/osx-only/autobuilder/no-xcode-with-spm/diagnostics.expected

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"markdownMessage": "A Swift package was detected, but no viable Xcode target was found.\n\nSwift Package Manager builds are not currently supported by `autobuild`. Set up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
2+
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/swift build --package-path <test-root-directory>\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
33
"severity": "error",
44
"source": {
55
"extractorName": "swift",
6-
"id": "swift/autobuilder/spm-not-supported",
7-
"name": "Swift Package Manager is not supported"
6+
"id": "swift/autobuilder/build-command-failed",
7+
"name": "Detected build command failed"
88
},
99
"visibility": {
1010
"cliSummaryTable": true,

‎swift/integration-tests/osx-only/autobuilder/only-tests-with-spm/diagnostics.expected

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"markdownMessage": "A Swift package was detected, but no viable Xcode target was found.\n\nSwift Package Manager builds are not currently supported by `autobuild`. Set up a [manual build command][1].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language",
2+
"markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/swift build --package-path <test-root-directory>\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs",
33
"severity": "error",
44
"source": {
55
"extractorName": "swift",
6-
"id": "swift/autobuilder/spm-not-supported",
7-
"name": "Swift Package Manager is not supported"
6+
"id": "swift/autobuilder/build-command-failed",
7+
"name": "Detected build command failed"
88
},
99
"visibility": {
1010
"cliSummaryTable": true,

‎swift/tools/autobuild.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22

33
if [[ "$OSTYPE" == "darwin"* ]]; then
4+
export CODEQL_SWIFT_CARTHAGE_EXEC=`which carthage`
5+
export CODEQL_SWIFT_POD_EXEC=`which pod`
46
exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/xcode-autobuilder"
57
else
68
exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder-incompatible-os"

‎swift/xcode-autobuilder/XcodeBuildRunner.cpp

Lines changed: 48 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,21 @@ static bool exec(const std::vector<std::string>& argv) {
5151
return true;
5252
}
5353

54-
void buildTarget(Target& target, bool dryRun) {
54+
static bool run_build_command(const std::vector<std::string>& argv, bool dryRun) {
55+
if (dryRun) {
56+
std::cout << absl::StrJoin(argv, " ") << "\n";
57+
} else {
58+
if (!exec(argv)) {
59+
DIAGNOSE_ERROR(buildCommandFailed,
60+
"`autobuild` failed to run the build command:\n\n```\n{}\n```",
61+
absl::StrJoin(argv, " "));
62+
return false;
63+
}
64+
}
65+
return true;
66+
}
67+
68+
bool buildXcodeTarget(const XcodeTarget& target, bool dryRun) {
5569
std::vector<std::string> argv({"/usr/bin/xcodebuild", "build"});
5670
if (!target.workspace.empty()) {
5771
argv.push_back("-workspace");
@@ -65,16 +79,40 @@ void buildTarget(Target& target, bool dryRun) {
6579
argv.push_back(target.name);
6680
argv.push_back("CODE_SIGNING_REQUIRED=NO");
6781
argv.push_back("CODE_SIGNING_ALLOWED=NO");
82+
return run_build_command(argv, dryRun);
83+
}
6884

69-
if (dryRun) {
70-
std::cout << absl::StrJoin(argv, " ") << "\n";
71-
} else {
72-
if (!exec(argv)) {
73-
DIAGNOSE_ERROR(buildCommandFailed,
74-
"`autobuild` failed to run the detected build command:\n\n```\n{}\n```",
75-
absl::StrJoin(argv, " "));
76-
codeql::Log::flush();
77-
exit(1);
85+
bool buildSwiftPackage(const std::filesystem::path& packageFile, bool dryRun) {
86+
std::vector<std::string> argv(
87+
{"/usr/bin/swift", "build", "--package-path", packageFile.parent_path()});
88+
return run_build_command(argv, dryRun);
89+
}
90+
91+
static void pod_install(const std::string& pod, const std::filesystem::path& podfile, bool dryRun) {
92+
std::vector<std::string> argv(
93+
{pod, "install", "--project-directory=" + podfile.parent_path().string()});
94+
run_build_command(argv, dryRun);
95+
}
96+
97+
static void carthage_install(const std::string& carthage,
98+
const std::filesystem::path& podfile,
99+
bool dryRun) {
100+
std::vector<std::string> argv(
101+
{carthage, "bootstrap", "--project-directory", podfile.parent_path()});
102+
run_build_command(argv, dryRun);
103+
}
104+
105+
void installDependencies(const ProjectStructure& target, bool dryRun) {
106+
auto pod = std::string(getenv("CODEQL_SWIFT_POD_EXEC") ?: "");
107+
auto carthage = std::string(getenv("CODEQL_SWIFT_CARTHAGE_EXEC") ?: "");
108+
if (!pod.empty() && !target.podfiles.empty()) {
109+
for (auto& podfile : target.podfiles) {
110+
pod_install(pod, podfile, dryRun);
111+
}
112+
}
113+
if (!carthage.empty() && !target.cartfiles.empty()) {
114+
for (auto& cartfile : target.cartfiles) {
115+
carthage_install(carthage, cartfile, dryRun);
78116
}
79117
}
80118
}

‎swift/xcode-autobuilder/XcodeBuildRunner.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
#pragma once
22

33
#include "swift/xcode-autobuilder/XcodeTarget.h"
4+
#include "swift/xcode-autobuilder/XcodeProjectParser.h"
5+
#include <filesystem>
46

5-
void buildTarget(Target& target, bool dryRun);
7+
void installDependencies(const ProjectStructure& target, bool dryRun);
8+
bool buildXcodeTarget(const XcodeTarget& target, bool dryRun);
9+
bool buildSwiftPackage(const std::filesystem::path& packageFile, bool dryRun);

‎swift/xcode-autobuilder/XcodeProjectParser.cpp

Lines changed: 33 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -198,14 +198,30 @@ static std::unordered_map<std::string, TargetData> mapTargetsToWorkspace(
198198
return targetMapping;
199199
}
200200

201-
static std::vector<fs::path> collectFiles(const std::string& workingDir) {
201+
struct ProjectFiles {
202+
std::vector<fs::path> xcodeFiles;
203+
std::vector<fs::path> packageFiles;
204+
std::vector<fs::path> podfiles;
205+
std::vector<fs::path> cartfiles;
206+
};
207+
208+
static ProjectFiles scanWorkingDir(const std::string& workingDir) {
209+
ProjectFiles structure;
202210
fs::path workDir(workingDir);
203211
std::vector<fs::path> files;
204212
auto end = fs::recursive_directory_iterator();
205213
for (auto it = fs::recursive_directory_iterator(workDir); it != end; ++it) {
206214
const auto& p = it->path();
207215
if (p.filename() == "Package.swift") {
208-
files.push_back(p);
216+
structure.packageFiles.push_back(p);
217+
continue;
218+
}
219+
if (p.filename() == "Podfile") {
220+
structure.podfiles.push_back(p);
221+
continue;
222+
}
223+
if (p.filename() == "Cartfile" || p.filename() == "Cartfile.private") {
224+
structure.cartfiles.push_back(p);
209225
continue;
210226
}
211227
if (!it->is_directory()) {
@@ -217,43 +233,31 @@ static std::vector<fs::path> collectFiles(const std::string& workingDir) {
217233
continue;
218234
}
219235
if (p.extension() == ".xcodeproj" || p.extension() == ".xcworkspace") {
220-
files.push_back(p);
236+
structure.xcodeFiles.push_back(p);
221237
}
222238
}
223-
return files;
239+
return structure;
224240
}
225241

226242
static std::unordered_map<std::string, std::vector<std::string>> collectWorkspaces(
227-
const std::string& workingDir,
228-
bool& swiftPackageEncountered) {
243+
const ProjectFiles& projectFiles) {
229244
// Here we are collecting list of all workspaces and Xcode projects corresponding to them
230245
// Projects without workspaces go into the same "empty-workspace" bucket
231-
swiftPackageEncountered = false;
232246
std::unordered_map<std::string, std::vector<std::string>> workspaces;
233247
std::unordered_set<std::string> projectsBelongingToWorkspace;
234-
std::vector<fs::path> files = collectFiles(workingDir);
235-
for (auto& path : files) {
248+
for (auto& path : projectFiles.xcodeFiles) {
236249
if (path.extension() == ".xcworkspace") {
237250
auto projects = readProjectsFromWorkspace(path.string());
238251
for (auto& project : projects) {
239252
projectsBelongingToWorkspace.insert(project.string());
240253
workspaces[path.string()].push_back(project.string());
241254
}
242-
} else if (!swiftPackageEncountered && path.filename() == "Package.swift") {
243-
// a package manifest must begin with a specific header comment
244-
// see https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html
245-
static constexpr std::string_view packageHeader = "// swift-tools-version:";
246-
std::array<char, packageHeader.size()> buffer{};
247-
std::string_view bufferView{buffer.data(), buffer.size()};
248-
if (std::ifstream{path}.read(buffer.data(), buffer.size()) && bufferView == packageHeader) {
249-
swiftPackageEncountered = true;
250-
}
251255
}
252256
}
253257
// Collect all projects not belonging to any workspace into a separate empty bucket
254-
for (auto& path : files) {
258+
for (auto& path : projectFiles.xcodeFiles) {
255259
if (path.extension() == ".xcodeproj") {
256-
if (projectsBelongingToWorkspace.count(path.string())) {
260+
if (projectsBelongingToWorkspace.contains(path.string())) {
257261
continue;
258262
}
259263
workspaces[std::string()].push_back(path.string());
@@ -262,11 +266,15 @@ static std::unordered_map<std::string, std::vector<std::string>> collectWorkspac
262266
return workspaces;
263267
}
264268

265-
Targets collectTargets(const std::string& workingDir) {
266-
Targets ret;
269+
ProjectStructure scanProjectStructure(const std::string& workingDir) {
270+
ProjectStructure ret;
267271
// Getting a list of workspaces and the project that belong to them
268-
auto workspaces = collectWorkspaces(workingDir, ret.swiftPackageEncountered);
272+
auto projectFiles = scanWorkingDir(workingDir);
273+
auto workspaces = collectWorkspaces(projectFiles);
269274
ret.xcodeEncountered = !workspaces.empty();
275+
ret.swiftPackages = std::move(projectFiles.packageFiles);
276+
ret.podfiles = std::move(projectFiles.podfiles);
277+
ret.cartfiles = std::move(projectFiles.cartfiles);
270278
if (!ret.xcodeEncountered) {
271279
return ret;
272280
}
@@ -278,8 +286,8 @@ Targets collectTargets(const std::string& workingDir) {
278286
auto targetFilesMapping = mapTargetsToSourceFiles(workspaces);
279287

280288
for (auto& [targetName, data] : targetMapping) {
281-
ret.targets.push_back(Target{data.workspace, data.project, targetName, data.type,
282-
targetFilesMapping[targetName]});
289+
ret.xcodeTargets.push_back(XcodeTarget{data.workspace, data.project, targetName, data.type,
290+
targetFilesMapping[targetName]});
283291
}
284292
return ret;
285293
}

‎swift/xcode-autobuilder/XcodeProjectParser.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,17 @@
33
#include "swift/xcode-autobuilder/XcodeTarget.h"
44
#include <vector>
55
#include <string>
6+
#include <filesystem>
67

7-
struct Targets {
8-
std::vector<Target> targets;
8+
struct ProjectStructure {
9+
std::vector<XcodeTarget> xcodeTargets;
910
bool xcodeEncountered;
10-
bool swiftPackageEncountered;
11+
// Swift Package Manager support
12+
std::vector<std::filesystem::path> swiftPackages;
13+
// CocoaPods support
14+
std::vector<std::filesystem::path> podfiles;
15+
// Carthage support
16+
std::vector<std::filesystem::path> cartfiles;
1117
};
1218

13-
Targets collectTargets(const std::string& workingDir);
19+
ProjectStructure scanProjectStructure(const std::string& workingDir);

0 commit comments

Comments
 (0)