Skip to content

Commit 2337c64

Browse files
authored
Native assets support for Linux (flutter#134031)
Support for FFI calls with `@Native external` functions through Native assets on Linux. This enables bundling native code without any build-system boilerplate code. For more info see: * flutter#129757 ### Implementation details for Linux. Mainly follows the design of flutter#130494. Some differences are: * Linux does not support cross compiling or compiling for multiple architectures, so this has not been implemented. * Linux has no add2app. The assets copying is done in the install-phase of the CMake build of a flutter app. CMake requires the native assets folder to exist, so we create it also when the feature is disabled or there are no assets. ### Tests This PR adds new tests to cover the various use cases. * packages/flutter_tools/test/general.shard/linux/native_assets_test.dart * Unit tests the Linux-specific part of building native assets. It also extends various existing tests: * packages/flutter_tools/test/integration.shard/native_assets_test.dart * Runs (incl hot reload/hot restart), builds, builds frameworks for Linux and flutter-tester.
1 parent d6d90b0 commit 2337c64

File tree

12 files changed

+919
-21
lines changed

12 files changed

+919
-21
lines changed

.ci.yaml

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -951,8 +951,10 @@ targets:
951951
{"dependency": "android_sdk", "version": "version:33v6"},
952952
{"dependency": "chrome_and_driver", "version": "version:115.0"},
953953
{"dependency": "clang", "version": "git_revision:5d5aba78dbbee75508f01bcaa69aedb2ab79065a"},
954-
{"dependency": "open_jdk", "version": "version:11"},
955-
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"}
954+
{"dependency": "cmake", "version": "build_id:8787856497187628321"},
955+
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"},
956+
{"dependency": "ninja", "version": "version:1.9.0"},
957+
{"dependency": "open_jdk", "version": "version:11"}
956958
]
957959
shard: tool_integration_tests
958960
subshard: "1_4"
@@ -975,8 +977,10 @@ targets:
975977
{"dependency": "android_sdk", "version": "version:33v6"},
976978
{"dependency": "chrome_and_driver", "version": "version:115.0"},
977979
{"dependency": "clang", "version": "git_revision:5d5aba78dbbee75508f01bcaa69aedb2ab79065a"},
978-
{"dependency": "open_jdk", "version": "version:11"},
979-
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"}
980+
{"dependency": "cmake", "version": "build_id:8787856497187628321"},
981+
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"},
982+
{"dependency": "ninja", "version": "version:1.9.0"},
983+
{"dependency": "open_jdk", "version": "version:11"}
980984
]
981985
shard: tool_integration_tests
982986
subshard: "2_4"
@@ -999,8 +1003,10 @@ targets:
9991003
{"dependency": "android_sdk", "version": "version:33v6"},
10001004
{"dependency": "chrome_and_driver", "version": "version:115.0"},
10011005
{"dependency": "clang", "version": "git_revision:5d5aba78dbbee75508f01bcaa69aedb2ab79065a"},
1002-
{"dependency": "open_jdk", "version": "version:11"},
1003-
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"}
1006+
{"dependency": "cmake", "version": "build_id:8787856497187628321"},
1007+
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"},
1008+
{"dependency": "ninja", "version": "version:1.9.0"},
1009+
{"dependency": "open_jdk", "version": "version:11"}
10041010
]
10051011
shard: tool_integration_tests
10061012
subshard: "3_4"
@@ -1023,8 +1029,10 @@ targets:
10231029
{"dependency": "android_sdk", "version": "version:33v6"},
10241030
{"dependency": "chrome_and_driver", "version": "version:115.0"},
10251031
{"dependency": "clang", "version": "git_revision:5d5aba78dbbee75508f01bcaa69aedb2ab79065a"},
1026-
{"dependency": "open_jdk", "version": "version:11"},
1027-
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"}
1032+
{"dependency": "cmake", "version": "build_id:8787856497187628321"},
1033+
{"dependency": "goldctl", "version": "git_revision:f808dcff91b221ae313e540c09d79696cd08b8de"},
1034+
{"dependency": "ninja", "version": "version:1.9.0"},
1035+
{"dependency": "open_jdk", "version": "version:11"}
10281036
]
10291037
shard: tool_integration_tests
10301038
subshard: "4_4"

packages/flutter_tools/lib/src/build_system/targets/native_assets.dart

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import '../../base/platform.dart';
1212
import '../../build_info.dart';
1313
import '../../dart/package_map.dart';
1414
import '../../ios/native_assets.dart';
15+
import '../../linux/native_assets.dart';
1516
import '../../macos/native_assets.dart';
1617
import '../../macos/xcode.dart';
1718
import '../../native_assets.dart';
@@ -118,6 +119,21 @@ class NativeAssets extends Target {
118119
fileSystem: fileSystem,
119120
buildRunner: buildRunner,
120121
);
122+
case TargetPlatform.linux_arm64:
123+
case TargetPlatform.linux_x64:
124+
final String? environmentBuildMode = environment.defines[kBuildMode];
125+
if (environmentBuildMode == null) {
126+
throw MissingDefineException(kBuildMode, name);
127+
}
128+
final BuildMode buildMode = BuildMode.fromCliName(environmentBuildMode);
129+
(_, dependencies) = await buildNativeAssetsLinux(
130+
targetPlatform: targetPlatform,
131+
buildMode: buildMode,
132+
projectUri: projectUri,
133+
yamlParentDirectory: environment.buildDir.uri,
134+
fileSystem: fileSystem,
135+
buildRunner: buildRunner,
136+
);
121137
case TargetPlatform.tester:
122138
if (const LocalPlatform().isMacOS) {
123139
(_, dependencies) = await buildNativeAssetsMacOS(
@@ -129,6 +145,15 @@ class NativeAssets extends Target {
129145
buildRunner: buildRunner,
130146
flutterTester: true,
131147
);
148+
} else if (const LocalPlatform().isLinux) {
149+
(_, dependencies) = await buildNativeAssetsLinux(
150+
buildMode: BuildMode.debug,
151+
projectUri: projectUri,
152+
yamlParentDirectory: environment.buildDir.uri,
153+
fileSystem: fileSystem,
154+
buildRunner: buildRunner,
155+
flutterTester: true,
156+
);
132157
} else {
133158
// TODO(dacoharkes): Implement other OSes. https://github.com/flutter/flutter/issues/129757
134159
// Write the file we claim to have in the [outputs].
@@ -142,8 +167,6 @@ class NativeAssets extends Target {
142167
case TargetPlatform.android:
143168
case TargetPlatform.fuchsia_arm64:
144169
case TargetPlatform.fuchsia_x64:
145-
case TargetPlatform.linux_arm64:
146-
case TargetPlatform.linux_x64:
147170
case TargetPlatform.web_javascript:
148171
case TargetPlatform.windows_x64:
149172
// TODO(dacoharkes): Implement other OSes. https://github.com/flutter/flutter/issues/129757

packages/flutter_tools/lib/src/linux/build_linux.dart

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import '../convert.dart';
1717
import '../flutter_plugins.dart';
1818
import '../globals.dart' as globals;
1919
import '../migrations/cmake_custom_command_migration.dart';
20+
import '../migrations/cmake_native_assets_migration.dart';
2021

2122
// Matches the following error and warning patterns:
2223
// - <file path>:<line>:<column>: (fatal) error: <error...>
@@ -45,6 +46,7 @@ Future<void> buildLinux(
4546

4647
final List<ProjectMigrator> migrators = <ProjectMigrator>[
4748
CmakeCustomCommandMigration(linuxProject, logger),
49+
CmakeNativeAssetsMigration(linuxProject, 'linux', logger),
4850
];
4951

5052
final ProjectMigration migration = ProjectMigration(migrators);
Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'package:native_assets_builder/native_assets_builder.dart' show BuildResult;
6+
import 'package:native_assets_cli/native_assets_cli.dart' hide BuildMode;
7+
import 'package:native_assets_cli/native_assets_cli.dart' as native_assets_cli;
8+
9+
import '../base/common.dart';
10+
import '../base/file_system.dart';
11+
import '../base/io.dart';
12+
import '../build_info.dart';
13+
import '../globals.dart' as globals;
14+
import '../native_assets.dart';
15+
16+
/// Dry run the native builds.
17+
///
18+
/// This does not build native assets, it only simulates what the final paths
19+
/// of all assets will be so that this can be embedded in the kernel file.
20+
Future<Uri?> dryRunNativeAssetsLinux({
21+
required NativeAssetsBuildRunner buildRunner,
22+
required Uri projectUri,
23+
bool flutterTester = false,
24+
required FileSystem fileSystem,
25+
}) async {
26+
if (await hasNoPackageConfig(buildRunner) || await isDisabledAndNoNativeAssets(buildRunner)) {
27+
return null;
28+
}
29+
30+
final Uri buildUri_ = nativeAssetsBuildUri(projectUri, OS.linux);
31+
final Iterable<Asset> nativeAssetPaths = await dryRunNativeAssetsLinuxInternal(
32+
fileSystem,
33+
projectUri,
34+
flutterTester,
35+
buildRunner,
36+
);
37+
final Uri nativeAssetsUri = await writeNativeAssetsYaml(
38+
nativeAssetPaths,
39+
buildUri_,
40+
fileSystem,
41+
);
42+
return nativeAssetsUri;
43+
}
44+
45+
Future<Iterable<Asset>> dryRunNativeAssetsLinuxInternal(
46+
FileSystem fileSystem,
47+
Uri projectUri,
48+
bool flutterTester,
49+
NativeAssetsBuildRunner buildRunner,
50+
) async {
51+
const OS targetOs = OS.linux;
52+
final Uri buildUri_ = nativeAssetsBuildUri(projectUri, targetOs);
53+
54+
globals.logger.printTrace('Dry running native assets for $targetOs.');
55+
final List<Asset> nativeAssets = (await buildRunner.dryRun(
56+
linkModePreference: LinkModePreference.dynamic,
57+
targetOs: targetOs,
58+
workingDirectory: projectUri,
59+
includeParentEnvironment: true,
60+
))
61+
.assets;
62+
ensureNoLinkModeStatic(nativeAssets);
63+
globals.logger.printTrace('Dry running native assets for $targetOs done.');
64+
final Uri? absolutePath = flutterTester ? buildUri_ : null;
65+
final Map<Asset, Asset> assetTargetLocations = _assetTargetLocations(nativeAssets, absolutePath);
66+
final Iterable<Asset> nativeAssetPaths = assetTargetLocations.values;
67+
return nativeAssetPaths;
68+
}
69+
70+
/// Builds native assets.
71+
///
72+
/// If [targetPlatform] is omitted, the current target architecture is used.
73+
///
74+
/// If [flutterTester] is true, absolute paths are emitted in the native
75+
/// assets mapping. This can be used for JIT mode without sandbox on the host.
76+
/// This is used in `flutter test` and `flutter run -d flutter-tester`.
77+
Future<(Uri? nativeAssetsYaml, List<Uri> dependencies)> buildNativeAssetsLinux({
78+
required NativeAssetsBuildRunner buildRunner,
79+
TargetPlatform? targetPlatform,
80+
required Uri projectUri,
81+
required BuildMode buildMode,
82+
bool flutterTester = false,
83+
Uri? yamlParentDirectory,
84+
required FileSystem fileSystem,
85+
}) async {
86+
const OS targetOs = OS.linux;
87+
final Uri buildUri_ = nativeAssetsBuildUri(projectUri, targetOs);
88+
final Directory buildDir = fileSystem.directory(buildUri_);
89+
if (!await buildDir.exists()) {
90+
// CMake requires the folder to exist to do copying.
91+
await buildDir.create(recursive: true);
92+
}
93+
if (await hasNoPackageConfig(buildRunner) || await isDisabledAndNoNativeAssets(buildRunner)) {
94+
final Uri nativeAssetsYaml = await writeNativeAssetsYaml(<Asset>[], yamlParentDirectory ?? buildUri_, fileSystem);
95+
return (nativeAssetsYaml, <Uri>[]);
96+
}
97+
98+
final Target target = targetPlatform != null ? _getNativeTarget(targetPlatform) : Target.current;
99+
final native_assets_cli.BuildMode buildModeCli = nativeAssetsBuildMode(buildMode);
100+
101+
globals.logger.printTrace('Building native assets for $target $buildModeCli.');
102+
final BuildResult result = await buildRunner.build(
103+
linkModePreference: LinkModePreference.dynamic,
104+
target: target,
105+
buildMode: buildModeCli,
106+
workingDirectory: projectUri,
107+
includeParentEnvironment: true,
108+
cCompilerConfig: await buildRunner.cCompilerConfig,
109+
);
110+
final List<Asset> nativeAssets = result.assets;
111+
final Set<Uri> dependencies = result.dependencies.toSet();
112+
ensureNoLinkModeStatic(nativeAssets);
113+
globals.logger.printTrace('Building native assets for $target done.');
114+
final Uri? absolutePath = flutterTester ? buildUri_ : null;
115+
final Map<Asset, Asset> assetTargetLocations = _assetTargetLocations(nativeAssets, absolutePath);
116+
await _copyNativeAssetsLinux(
117+
buildUri_,
118+
assetTargetLocations,
119+
buildMode,
120+
fileSystem,
121+
);
122+
final Uri nativeAssetsUri = await writeNativeAssetsYaml(
123+
assetTargetLocations.values,
124+
yamlParentDirectory ?? buildUri_,
125+
fileSystem,
126+
);
127+
return (nativeAssetsUri, dependencies.toList());
128+
}
129+
130+
Map<Asset, Asset> _assetTargetLocations(
131+
List<Asset> nativeAssets,
132+
Uri? absolutePath,
133+
) =>
134+
<Asset, Asset>{
135+
for (final Asset asset in nativeAssets) asset: _targetLocationLinux(asset, absolutePath),
136+
};
137+
138+
Asset _targetLocationLinux(Asset asset, Uri? absolutePath) {
139+
final AssetPath path = asset.path;
140+
switch (path) {
141+
case AssetSystemPath _:
142+
case AssetInExecutable _:
143+
case AssetInProcess _:
144+
return asset;
145+
case AssetAbsolutePath _:
146+
final String fileName = path.uri.pathSegments.last;
147+
Uri uri;
148+
if (absolutePath != null) {
149+
// Flutter tester needs full host paths.
150+
uri = absolutePath.resolve(fileName);
151+
} else {
152+
// Flutter Desktop needs "absolute" paths inside the app.
153+
// "relative" in the context of native assets would be relative to the
154+
// kernel or aot snapshot.
155+
uri = Uri(path: fileName);
156+
}
157+
return asset.copyWith(path: AssetAbsolutePath(uri));
158+
}
159+
throw Exception('Unsupported asset path type ${path.runtimeType} in asset $asset');
160+
}
161+
162+
/// Extract the [Target] from a [TargetPlatform].
163+
Target _getNativeTarget(TargetPlatform targetPlatform) {
164+
switch (targetPlatform) {
165+
case TargetPlatform.linux_x64:
166+
return Target.linuxX64;
167+
case TargetPlatform.linux_arm64:
168+
return Target.linuxArm64;
169+
case TargetPlatform.android:
170+
case TargetPlatform.ios:
171+
case TargetPlatform.darwin:
172+
case TargetPlatform.windows_x64:
173+
case TargetPlatform.fuchsia_arm64:
174+
case TargetPlatform.fuchsia_x64:
175+
case TargetPlatform.tester:
176+
case TargetPlatform.web_javascript:
177+
case TargetPlatform.android_arm:
178+
case TargetPlatform.android_arm64:
179+
case TargetPlatform.android_x64:
180+
case TargetPlatform.android_x86:
181+
throw Exception('Unknown targetPlatform: $targetPlatform.');
182+
}
183+
}
184+
185+
Future<void> _copyNativeAssetsLinux(
186+
Uri buildUri,
187+
Map<Asset, Asset> assetTargetLocations,
188+
BuildMode buildMode,
189+
FileSystem fileSystem,
190+
) async {
191+
if (assetTargetLocations.isNotEmpty) {
192+
globals.logger.printTrace('Copying native assets to ${buildUri.toFilePath()}.');
193+
final Directory buildDir = fileSystem.directory(buildUri.toFilePath());
194+
if (!buildDir.existsSync()) {
195+
buildDir.createSync(recursive: true);
196+
}
197+
for (final MapEntry<Asset, Asset> assetMapping in assetTargetLocations.entries) {
198+
final Uri source = (assetMapping.key.path as AssetAbsolutePath).uri;
199+
final Uri target = (assetMapping.value.path as AssetAbsolutePath).uri;
200+
final Uri targetUri = buildUri.resolveUri(target);
201+
final String targetFullPath = targetUri.toFilePath();
202+
await fileSystem.file(source).copy(targetFullPath);
203+
}
204+
globals.logger.printTrace('Copying native assets done.');
205+
}
206+
}
207+
208+
/// Flutter expects `clang++` to be on the path on Linux hosts.
209+
///
210+
/// Search for the accompanying `clang`, `ar`, and `ld`.
211+
Future<CCompilerConfig> cCompilerConfigLinux() async {
212+
const String kClangPlusPlusBinary = 'clang++';
213+
const String kClangBinary = 'clang';
214+
const String kArBinary = 'llvm-ar';
215+
const String kLdBinary = 'ld.lld';
216+
217+
final ProcessResult whichResult = await globals.processManager.run(<String>['which', kClangPlusPlusBinary]);
218+
if (whichResult.exitCode != 0) {
219+
throwToolExit('Failed to find $kClangPlusPlusBinary on PATH.');
220+
}
221+
File clangPpFile = globals.fs.file((whichResult.stdout as String).trim());
222+
clangPpFile = globals.fs.file(await clangPpFile.resolveSymbolicLinks());
223+
224+
final Directory clangDir = clangPpFile.parent;
225+
final Map<String, Uri> binaryPaths = <String, Uri>{};
226+
for (final String binary in <String>[kClangBinary, kArBinary, kLdBinary]) {
227+
final File binaryFile = clangDir.childFile(binary);
228+
if (!await binaryFile.exists()) {
229+
throwToolExit("Failed to find $binary relative to $clangPpFile: $binaryFile doesn't exist.");
230+
}
231+
binaryPaths[binary] = binaryFile.uri;
232+
}
233+
return CCompilerConfig(
234+
ar: binaryPaths[kArBinary],
235+
cc: binaryPaths[kClangBinary],
236+
ld: binaryPaths[kLdBinary],
237+
);
238+
}

0 commit comments

Comments
 (0)