[vm,dartdev] Support dynamic linking between libraries

The same bundling that is used for `dart build` is now also used for
`dart test` and `dart run`, except that the output directory is
`.dart_tool/native_assets`. This way all native code assets are placed
next to each other in the `lib` directory, and loaded from there
instead of loading them in place from where the build/link hooks
placed them. By standardizing on this layout the different modes of
running dart code that support native assets can use the same
mechanisms to support dynamic linking between libraries.

On macOS, install names of dylibs are rewritten to support dynamic
linking, similar to the changes in
flutter/flutter#153054.

On Windows, loading of DLLs is altered so that the directory of the DLL
that is being loaded is considered when loading dependent DLLs.

Tests are added to verify that dynamic linking works as expected.

TEST=pkg/dartdev/test/native_assets/{build,run,test}_test.dart

[email protected]

Related: dart-lang/native#190
Fixes: #56459
Change-Id: Ie4a41e5b7382ab1cea39e93d29d085bf9986828b
Cq-Include-Trybots: luci.dart.try:pkg-linux-debug-try,pkg-linux-release-arm64-try,pkg-linux-release-try,pkg-mac-release-arm64-try,pkg-mac-release-try,pkg-win-release-arm64-try,pkg-win-release-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/381580
Reviewed-by: Moritz Sümmermann <[email protected]>
Commit-Queue: Daco Harkes <[email protected]>
Reviewed-by: Daco Harkes <[email protected]>

Author: blaugold

pkg/dartdev/lib/src/commands/build.dart

@@ -8,6 +8,7 @@ import 'dart:io';
 import 'package:dart2native/generate.dart';
 import 'package:dartdev/src/commands/compile.dart';
 import 'package:dartdev/src/experiments.dart';
+import 'package:dartdev/src/native_assets_bundling.dart';
 import 'package:dartdev/src/sdk.dart';
 import 'package:dartdev/src/utils.dart';
 import 'package:front_end/src/api_prototype/compiler_options.dart'
@@ -21,9 +22,6 @@ import 'package:vm/target_os.dart'; // For possible --target-os values.
 import '../core.dart';
 import '../native_assets.dart';
 
-const _libOutputDirectory = 'lib';
-const _dataOutputDirectory = 'assets';
-
 class BuildCommand extends DartdevCommand {
   static const String cmdName = 'build';
   static const String outputOptionName = 'output';
@@ -240,60 +238,34 @@ class BuildCommand extends DartdevCommand {
         return 255;
       }
 
-      final tempUri = tempDir.uri;
-      Uri? assetsDartUri;
       final allAssets = linkResult.encodedAssets;
-      final dataAssets = allAssets
-          .where((e) => e.type == DataAsset.type)
-          .map(DataAsset.fromEncoded)
-          .toList();
-      final codeAssets = allAssets
+
+      final staticAssets = allAssets
           .where((e) => e.type == CodeAsset.type)
           .map(CodeAsset.fromEncoded)
-          .toList();
-
-      final staticAssets =
-          codeAssets.where((e) => e.linkMode == StaticLinking());
+          .where((e) => e.linkMode == StaticLinking());
       if (staticAssets.isNotEmpty) {
         stderr.write(
             """'dart build' does not yet support CodeAssets with static linking.
 Use linkMode as dynamic library instead.""");
         return 255;
       }
-      if (allAssets.isNotEmpty) {
-        final kernelAssets = <KernelAsset>[];
-        final filesToCopy = <(String id, Uri, KernelAssetRelativePath)>[];
 
-        for (final asset in codeAssets) {
-          final kernelAsset = asset.targetLocation(target);
-          kernelAssets.add(kernelAsset);
-          final targetPath = kernelAsset.path;
-          if (targetPath is KernelAssetRelativePath) {
-            filesToCopy.add((asset.id, asset.file!, targetPath));
-          }
-        }
-        for (final asset in dataAssets) {
-          final kernelAsset = asset.targetLocation(target);
-          kernelAssets.add(kernelAsset);
-          final targetPath = kernelAsset.path;
-          if (targetPath is KernelAssetRelativePath) {
-            filesToCopy.add((asset.id, asset.file, targetPath));
-          }
-        }
-        assetsDartUri = await _writeAssetsYaml(
-          kernelAssets,
-          assetsDartUri,
-          tempUri,
+      Uri? nativeAssetsYamlUri;
+      if (allAssets.isNotEmpty) {
+        final kernelAssets = await bundleNativeAssets(
+          allAssets,
+          target,
+          outputUri,
+          relocatable: true,
+          verbose: true,
         );
-        if (allAssets.isNotEmpty) {
-          stdout.writeln(
-              'Copying ${filesToCopy.length} build assets: ${filesToCopy.map((e) => e.$1)}');
-          _copyAssets(filesToCopy, outputUri);
-        }
+        nativeAssetsYamlUri =
+            await writeNativeAssetsYaml(kernelAssets, tempDir.uri);
       }
 
       await snapshotGenerator.generate(
-        nativeAssets: assetsDartUri?.toFilePath(),
+        nativeAssets: nativeAssetsYamlUri?.toFilePath(),
       );
 
       // End linking here.
@@ -302,29 +274,6 @@ Use linkMode as dynamic library instead.""");
     }
     return 0;
   }
-
-  void _copyAssets(
-    List<(String id, Uri, KernelAssetRelativePath)> assetTargetLocations,
-    Uri output,
-  ) {
-    for (final (_, file, targetPath) in assetTargetLocations) {
-      file.copyTo(targetPath, output);
-    }
-  }
-
-  Future<Uri> _writeAssetsYaml(
-    List<KernelAsset> assetTargetLocations,
-    Uri? nativeAssetsDartUri,
-    Uri tempUri,
-  ) async {
-    stdout.writeln('Writing native_assets.yaml.');
-    nativeAssetsDartUri = tempUri.resolve('native_assets.yaml');
-    final assetsContent =
-        KernelAssets(assetTargetLocations).toNativeAssetsFile();
-    await Directory.fromUri(nativeAssetsDartUri.resolve('.')).create();
-    await File(nativeAssetsDartUri.toFilePath()).writeAsString(assetsContent);
-    return nativeAssetsDartUri;
-  }
 }
 
 extension on String {
@@ -333,57 +282,6 @@ extension on String {
   String removeDotDart() => replaceFirst(RegExp(r'\.dart$'), '');
 }
 
-extension on Uri {
-  void copyTo(KernelAssetRelativePath target, Uri outputUri) {
-    if (this != target.uri) {
-      final targetUri = outputUri.resolveUri(target.uri);
-      File.fromUri(targetUri).createSync(
-        recursive: true,
-        exclusive: true,
-      );
-      File.fromUri(this).copySync(targetUri.toFilePath());
-    }
-  }
-}
-
-extension on CodeAsset {
-  KernelAsset targetLocation(Target target) {
-    final KernelAssetPath kernelAssetPath;
-    switch (linkMode) {
-      case DynamicLoadingSystem dynamicLoading:
-        kernelAssetPath = KernelAssetSystemPath(dynamicLoading.uri);
-      case LookupInExecutable _:
-        kernelAssetPath = KernelAssetInExecutable();
-      case LookupInProcess _:
-        kernelAssetPath = KernelAssetInProcess();
-      case DynamicLoadingBundled _:
-        kernelAssetPath = KernelAssetRelativePath(
-          Uri(path: path.join(_libOutputDirectory, file!.pathSegments.last)),
-        );
-      default:
-        throw Exception(
-          'Unsupported CodeAsset linkMode ${linkMode.runtimeType} in asset $this',
-        );
-    }
-    return KernelAsset(
-      id: id,
-      target: target,
-      path: kernelAssetPath,
-    );
-  }
-}
-
-extension on DataAsset {
-  KernelAsset targetLocation(Target target) {
-    return KernelAsset(
-      id: id,
-      target: target,
-      path: KernelAssetRelativePath(
-          Uri(path: path.join(_dataOutputDirectory, file.pathSegments.last))),
-    );
-  }
-}
-
 // TODO(https://github.com/dart-lang/package_config/issues/126): Expose this
 // logic in package:package_config.
 Future<Uri?> packageConfigUri(Uri uri) async {

pkg/dartdev/lib/src/native_assets.dart

@@ -5,6 +5,7 @@
 import 'dart:async';
 import 'dart:io';
 
+import 'package:dartdev/src/native_assets_bundling.dart';
 import 'package:dartdev/src/sdk.dart';
 import 'package:dartdev/src/utils.dart';
 import 'package:logging/logging.dart';
@@ -100,62 +101,24 @@ Future<Uri?> compileNativeAssetsJitYamlFile({
     runPackageName: runPackageName,
   );
   if (assets == null) return null;
-  final codeAssets = assets
-      .where((e) => e.type == CodeAsset.type)
-      .map(CodeAsset.fromEncoded)
-      .toList();
-  final dataAssets = assets
-      .where((e) => e.type == DataAsset.type)
-      .map(DataAsset.fromEncoded)
-      .toList();
-  final kernelAssets = KernelAssets([
-    ...[
-      for (final asset in codeAssets) _targetLocation(asset),
-    ],
-    ...[
-      for (final asset in dataAssets) _dataTargetLocation(asset),
-    ]
-  ]);
 
-  final workingDirectory = Directory.current.uri;
-  final assetsUri = workingDirectory.resolve('.dart_tool/native_assets.yaml');
-  final nativeAssetsYaml = '''# Native assets mapping for host OS in JIT mode.
-# Generated by dartdev and package:native_assets_builder.
-${kernelAssets.toNativeAssetsFile()}''';
-  final assetFile = File(assetsUri.toFilePath());
-  await assetFile.writeAsString(nativeAssetsYaml);
-  return assetsUri;
-}
+  final dartToolUri = Directory.current.uri.resolve('.dart_tool/');
+  final outputUri = dartToolUri.resolve('native_assets/');
+  await Directory.fromUri(outputUri).create(recursive: true);
 
-KernelAsset _targetLocation(CodeAsset asset) {
-  final linkMode = asset.linkMode;
-  final KernelAssetPath kernelAssetPath;
-  switch (linkMode) {
-    case DynamicLoadingSystem _:
-      kernelAssetPath = KernelAssetSystemPath(linkMode.uri);
-    case LookupInExecutable _:
-      kernelAssetPath = KernelAssetInExecutable();
-    case LookupInProcess _:
-      kernelAssetPath = KernelAssetInProcess();
-    case DynamicLoadingBundled _:
-      kernelAssetPath = KernelAssetAbsolutePath(asset.file!);
-    default:
-      throw Exception(
-        'Unsupported CodeAsset linkMode ${linkMode.runtimeType} in asset $asset',
-      );
-  }
-  return KernelAsset(
-    id: asset.id,
-    target: Target.fromArchitectureAndOS(asset.architecture!, asset.os),
-    path: kernelAssetPath,
+  final kernelAssets = await bundleNativeAssets(
+    assets,
+    Target.current,
+    outputUri,
+    relocatable: false,
   );
-}
 
-KernelAsset _dataTargetLocation(DataAsset asset) {
-  return KernelAsset(
-    id: asset.id,
-    target: Target.current,
-    path: KernelAssetAbsolutePath(asset.file),
+  return await writeNativeAssetsYaml(
+    kernelAssets,
+    dartToolUri,
+    header: '''# Native assets mapping for host OS in JIT mode.
+# Generated by dartdev and package:native_assets_builder.
+''',
   );
 }