Dartdev cross compilation support

- Add `target-arch` option to `dart compile exe|aot-snapshot`
- Add artifacts cache in `~/.dart`, and download gen_snapshot and
  dartaotruntime from cloud storage if necessary

I've manually built an aot snapshot and exe on macOS ARM64 for Linux
X64, copied them over to a linux machine, and made sure they work.

Change-Id: I74dd581ecb573ff4069a2fe19f9d04aaa352538e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/417801
Reviewed-by: Tess Strickland <[email protected]>
Commit-Queue: Ivan Inozemtsev <[email protected]>
Reviewed-by: Ben Konyi <[email protected]>

Author: iinozemtsev

pkg/dart2native/lib/dart2native.dart

@@ -7,6 +7,7 @@ import 'dart:typed_data';
 
 import 'package:collection/collection.dart';
 import 'package:kernel/binary/tag.dart' show Tag;
+import 'package:native_assets_cli/code_assets_builder.dart' show OS;
 import 'package:path/path.dart' as path;
 
 import 'dart2native_macho.dart' show writeAppendedMachOExecutable;
@@ -19,7 +20,7 @@ final genKernel = path.join(
   'snapshots',
   'gen_kernel_aot.dart.snapshot',
 );
-final genSnapshot = path.join(
+final genSnapshotHost = path.join(
   binDir.path,
   'utils',
   'gen_snapshot$executableSuffix',
@@ -65,36 +66,38 @@ Future<bool> isKernelFile(String path) async {
 // WARNING: this method is used within google3, so don't try to refactor so
 // [dartaotruntime] is a constant inside this file.
 Future<void> writeAppendedExecutable(
-  String dartaotruntime,
+  String dartAotRuntime,
   String payloadPath,
   String outputPath,
+  OS targetOS,
 ) async {
-  if (Platform.isMacOS) {
-    return await writeAppendedMachOExecutable(
-        dartaotruntime, payloadPath, outputPath);
-  } else if (Platform.isWindows) {
-    return await writeAppendedPortableExecutable(
-        dartaotruntime, payloadPath, outputPath);
+  switch (targetOS) {
+    case OS.macOS:
+      return await writeAppendedMachOExecutable(
+          dartAotRuntime, payloadPath, outputPath);
+    case OS.windows:
+      return await writeAppendedPortableExecutable(
+          dartAotRuntime, payloadPath, outputPath);
+    default:
+      final dartAotRuntimeFile = File(dartAotRuntime);
+      final dartAotRuntimeLength = dartAotRuntimeFile.lengthSync();
+
+      final padding = (elfPageSize - dartAotRuntimeLength) % elfPageSize;
+      final padBytes = Uint8List(padding);
+      final offset = dartAotRuntimeLength + padding;
+
+      // Note: The offset is always Little Endian regardless of host.
+      final offsetBytes = ByteData(8) // 64 bit in bytes.
+        ..setUint64(0, offset, Endian.little);
+
+      final outputFile = File(outputPath).openWrite();
+      outputFile.add(dartAotRuntimeFile.readAsBytesSync());
+      outputFile.add(padBytes);
+      outputFile.add(File(payloadPath).readAsBytesSync());
+      outputFile.add(offsetBytes.buffer.asUint8List());
+      outputFile.add(appJitMagicNumber);
+      await outputFile.close();
   }
-
-  final dartaotruntimeFile = File(dartaotruntime);
-  final dartaotruntimeLength = dartaotruntimeFile.lengthSync();
-
-  final padding = (elfPageSize - dartaotruntimeLength) % elfPageSize;
-  final padBytes = Uint8List(padding);
-  final offset = dartaotruntimeLength + padding;
-
-  // Note: The offset is always Little Endian regardless of host.
-  final offsetBytes = ByteData(8) // 64 bit in bytes.
-    ..setUint64(0, offset, Endian.little);
-
-  final outputFile = File(outputPath).openWrite();
-  outputFile.add(dartaotruntimeFile.readAsBytesSync());
-  outputFile.add(padBytes);
-  outputFile.add(File(payloadPath).readAsBytesSync());
-  outputFile.add(offsetBytes.buffer.asUint8List());
-  outputFile.add(appJitMagicNumber);
-  await outputFile.close();
 }
 
 Future<ProcessResult> markExecutable(String outputFile) {
@@ -106,13 +109,13 @@ Future<ProcessResult> markExecutable(String outputFile) {
 /// Also takes a path to the [recordedUsagesFile] JSON file, where the method
 /// calls to static functions annotated with `@RecordUse` will be collected.
 Future<ProcessResult> generateKernelHelper({
-  required String dartaotruntime,
+  required String hostDartAotRuntime,
   String? sourceFile,
   required String kernelFile,
   String? packages,
   List<String> defines = const [],
   String enableExperiment = '',
-  String? targetOS,
+  OS? targetOS,
   List<String> extraGenKernelOptions = const [],
   String? nativeAssets,
   String? recordedUsagesFile,
@@ -145,10 +148,11 @@ Future<ProcessResult> generateKernelHelper({
     ...extraGenKernelOptions,
     if (sourceFile != null) sourceFile,
   ];
-  return Process.run(dartaotruntime, args);
+  return Process.run(hostDartAotRuntime, args);
 }
 
 Future<ProcessResult> generateAotSnapshotHelper(
+    String genSnapshot,
     String kernelFile,
     String snapshotFile,
     String? debugFile,

pkg/dart2native/lib/generate.dart

@@ -4,14 +4,15 @@
 
 import 'dart:io';
 
+import 'package:native_assets_cli/code_assets_builder.dart' show OS;
 import 'package:path/path.dart' as path;
 
 import 'dart2native.dart';
 import 'src/generate_utils.dart';
 
-export 'dart2native.dart' show genKernel, genSnapshot;
+export 'dart2native.dart' show genKernel, genSnapshotHost;
 
-final dartaotruntime = path.join(
+final hostDartAotRuntime = path.join(
   binDir.path,
   'dartaotruntime$executableSuffix',
 );
@@ -34,20 +35,24 @@ enum Kind {
 /// See also the docs for [_Generator].
 extension type KernelGenerator._(_Generator _generator) {
   KernelGenerator({
+    required String genSnapshot,
+    required String targetDartAotRuntime,
     required String sourceFile,
     required List<String> defines,
     Kind kind = Kind.exe,
     String? outputFile,
     String? debugFile,
     String? packages,
-    String? targetOS,
+    OS? targetOS,
     String? depFile,
     String enableExperiment = '',
     bool enableAsserts = false,
     bool verbose = false,
     String verbosity = 'all',
     required Directory tempDir,
   }) : _generator = _Generator(
+          genSnapshot: genSnapshot,
+          targetDartAotRuntime: targetDartAotRuntime,
           sourceFile: sourceFile,
           defines: defines,
           tempDir: tempDir,
@@ -123,7 +128,7 @@ class _Generator {
   /// Specifies the operating system the executable is being generated for. This
   /// must be provided when [_kind] is [Kind.exe], and it must match the current
   /// operating system.
-  final String? _targetOS;
+  final OS? _targetOS;
 
   /// A comma separated list of language experiments to be enabled.
   final String _enableExperiment;
@@ -153,14 +158,22 @@ class _Generator {
   /// The path to the [depfile](https://ninja-build.org/manual.html#_depfile).
   final String? _depFile;
 
+  /// The path to the `gen_snapshot` tool.
+  final String _genSnapshot;
+
+  /// The path to the `dartaotruntime` for a target platform.
+  final String _targetDartAotRuntime;
+
   _Generator({
+    required String genSnapshot,
+    required String targetDartAotRuntime,
     required String sourceFile,
     required List<String> defines,
     required Kind kind,
     String? outputFile,
     String? debugFile,
     String? packages,
-    String? targetOS,
+    OS? targetOS,
     String? depFile,
     required String enableExperiment,
     required bool enableAsserts,
@@ -180,13 +193,12 @@ class _Generator {
         _depFile = depFile,
         _programKernelFile = path.join(tempDir.path, 'program.dill'),
         _sourcePath = _normalize(sourceFile)!,
-        _packages = _normalize(packages) {
+        _packages = _normalize(packages),
+        _genSnapshot = genSnapshot,
+        _targetDartAotRuntime = targetDartAotRuntime {
     if (_kind == Kind.exe) {
       if (_targetOS == null) {
         throw ArgumentError('targetOS must be specified for executables.');
-      } else if (_targetOS != Platform.operatingSystem) {
-        throw UnsupportedError(
-            'Cross compilation not supported for executables.');
       }
     }
   }
@@ -203,7 +215,7 @@ class _Generator {
     }
 
     final kernelResult = await generateKernelHelper(
-      dartaotruntime: dartaotruntime,
+      hostDartAotRuntime: hostDartAotRuntime,
       sourceFile: _sourcePath,
       kernelFile: _programKernelFile,
       packages: _packages,
@@ -256,12 +268,13 @@ class _Generator {
 
     if (_verbose) {
       print('Compiling $_sourcePath to $outputPath using format $_kind:');
-      print('Generating AOT snapshot. $genSnapshot $extraOptions');
+      print('Generating AOT snapshot. $_genSnapshot $extraOptions');
     }
     final snapshotFile = _kind == Kind.aot
         ? outputPath
         : path.join(_tempDir.path, 'snapshot.aot');
     final snapshotResult = await generateAotSnapshotHelper(
+      _genSnapshot,
       kernelFile,
       snapshotFile,
       debugPath,
@@ -280,7 +293,8 @@ class _Generator {
       if (_verbose) {
         print('Generating executable.');
       }
-      await writeAppendedExecutable(dartaotruntime, snapshotFile, outputPath);
+      await writeAppendedExecutable(
+          _targetDartAotRuntime, snapshotFile, outputPath, _targetOS!);
 
       if (Platform.isLinux || Platform.isMacOS) {
         if (_verbose) {
@@ -302,7 +316,7 @@ class _Generator {
       final nativeAssetsDillFile =
           path.join(_tempDir.path, 'native_assets.dill');
       final kernelResult = await generateKernelHelper(
-        dartaotruntime: dartaotruntime,
+        hostDartAotRuntime: hostDartAotRuntime,
         kernelFile: nativeAssetsDillFile,
         packages: _packages,
         defines: _defines,
@@ -391,7 +405,7 @@ Future<void> generateKernel({
   packages = _normalize(packages);
 
   final kernelResult = await generateKernelHelper(
-    dartaotruntime: dartaotruntime,
+    hostDartAotRuntime: hostDartAotRuntime,
     sourceFile: sourcePath,
     kernelFile: outputPath,
     packages: packages,

pkg/dart2native/pubspec.yaml

@@ -13,9 +13,10 @@ executables:
 
 # Use 'any' constraints here; we get our versions from the DEPS file.
 dependencies:
-   collection: any
-   kernel: any
-   path: any
+  collection: any
+  kernel: any
+  native_assets_cli: any
+  path: any
 
 # Use 'any' constraints here; we get our versions from the DEPS file.
 dev_dependencies:

pkg/dartdev/lib/src/commands/build.dart

@@ -63,7 +63,7 @@ class BuildCommand extends DartdevCommand {
   @override
   Future<int> run() async {
     if (!Sdk.checkArtifactExists(genKernel) ||
-        !Sdk.checkArtifactExists(genSnapshot) ||
+        !Sdk.checkArtifactExists(genSnapshotHost) ||
         !Sdk.checkArtifactExists(sdk.dart)) {
       return 255;
     }
@@ -152,14 +152,16 @@ class BuildCommand extends DartdevCommand {
         recordedUsagesPath = path.join(tempDir.path, 'recorded_usages.json');
       }
       final generator = KernelGenerator(
+        genSnapshot: genSnapshotHost,
+        targetDartAotRuntime: hostDartAotRuntime,
         kind: format,
         sourceFile: sourceUri.toFilePath(),
         outputFile: outputExeUri.toFilePath(),
         verbose: verbose,
         verbosity: args.option('verbosity')!,
         defines: [],
         packages: packageConfig.toFilePath(),
-        targetOS: targetOS,
+        targetOS: targetOS == null ? null : OS.fromString(targetOS),
         enableExperiment: args.enabledExperiments.join(','),
         tempDir: tempDir,
       );