diff --git a/build_modules/lib/build_modules.dart b/build_modules/lib/build_modules.dart index 834333bb4d..332155c3aa 100644 --- a/build_modules/lib/build_modules.dart +++ b/build_modules/lib/build_modules.dart @@ -2,17 +2,23 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +export 'src/ddc_names.dart'; export 'src/errors.dart' show MissingModulesException, UnsupportedModules; -export 'src/kernel_builder.dart' - show KernelBuilder, multiRootScheme, reportUnusedKernelInputs; +export 'src/kernel_builder.dart' show KernelBuilder, reportUnusedKernelInputs; export 'src/meta_module_builder.dart' show MetaModuleBuilder, metaModuleExtension; export 'src/meta_module_clean_builder.dart' show MetaModuleCleanBuilder, metaModuleCleanExtension; export 'src/module_builder.dart' show ModuleBuilder, moduleExtension; +export 'src/module_library.dart' show ModuleLibrary; export 'src/module_library_builder.dart' show ModuleLibraryBuilder, moduleLibraryExtension; export 'src/modules.dart'; export 'src/platform.dart' show DartPlatform; export 'src/scratch_space.dart' show scratchSpace, scratchSpaceResource; -export 'src/workers.dart' show dartdevkDriverResource, maxWorkersPerTask; +export 'src/workers.dart' + show + dartdevkDriverResource, + frontendServerProxyDriverResource, + maxWorkersPerTask, + persistentFrontendServerResource; diff --git a/build_modules/lib/src/common.dart b/build_modules/lib/src/common.dart index 1d90fdcf1e..7c8c8792b1 100644 --- a/build_modules/lib/src/common.dart +++ b/build_modules/lib/src/common.dart @@ -2,9 +2,17 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + import 'package:build/build.dart'; +import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; +const multiRootScheme = 'org-dartlang-app'; +const webHotReloadOption = 'web-hot-reload'; +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); +final packagesFilePath = p.join('.dart_tool', 'package_config.json'); + final defaultAnalysisOptionsId = AssetId( 'build_modules', 'lib/src/analysis_options.default.yaml', @@ -16,6 +24,12 @@ String defaultAnalysisOptionsArg(ScratchSpace scratchSpace) => enum ModuleStrategy { fine, coarse } ModuleStrategy moduleStrategy(BuilderOptions options) { + // DDC's Library Bundle module system only supports fine modules since it must + // align with the Frontend Server's library management scheme. + final usesWebHotReload = options.config[webHotReloadOption] as bool? ?? false; + if (usesWebHotReload) { + return ModuleStrategy.fine; + } final config = options.config['strategy'] as String? ?? 'coarse'; switch (config) { case 'coarse': diff --git a/build_web_compilers/lib/src/ddc_names.dart b/build_modules/lib/src/ddc_names.dart similarity index 76% rename from build_web_compilers/lib/src/ddc_names.dart rename to build_modules/lib/src/ddc_names.dart index 5664eaf0c8..fbcf80a10a 100644 --- a/build_web_compilers/lib/src/ddc_names.dart +++ b/build_modules/lib/src/ddc_names.dart @@ -1,14 +1,25 @@ -// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'package:path/path.dart' as p; +/// Logic in this file must be synchronized with their namesakes in DDC at: +/// pkg/dev_compiler/lib/src/compiler/js_names.dart + +bool isSdkInternalRuntimeUri(Uri importUri) { + return importUri.isScheme('dart') && importUri.path == '_runtime'; +} + +String libraryUriToJsIdentifier(Uri importUri) { + if (importUri.isScheme('dart')) { + return isSdkInternalRuntimeUri(importUri) ? 'dart' : importUri.path; + } + return pathToJSIdentifier(p.withoutExtension(importUri.pathSegments.last)); +} + /// Transforms a path to a valid JS identifier. /// -/// This logic must be synchronized with [pathToJSIdentifier] in DDC at: -/// pkg/dev_compiler/lib/src/compiler/module_builder.dart -/// /// For backwards compatibility, if this pattern is changed, /// dev_compiler_bootstrap.dart must be updated to accept both old and new /// patterns. @@ -35,12 +46,11 @@ String toJSIdentifier(String name) { for (var i = 0; i < name.length; i++) { final ch = name[i]; final needsEscape = ch == r'$' || _invalidCharInIdentifier.hasMatch(ch); - if (needsEscape && buffer == null) { - buffer = StringBuffer(name.substring(0, i)); - } - if (buffer != null) { - buffer.write(needsEscape ? '\$${ch.codeUnits.join("")}' : ch); + if (needsEscape) { + buffer ??= StringBuffer(name.substring(0, i)); } + + buffer?.write(needsEscape ? '\$${ch.codeUnits.join("")}' : ch); } final result = buffer != null ? '$buffer' : name; @@ -56,7 +66,11 @@ String toJSIdentifier(String name) { /// Also handles invalid variable names in strict mode, like "arguments". bool invalidVariableName(String keyword, {bool strictMode = true}) { switch (keyword) { - // http://www.ecma-international.org/ecma-262/6.0/#sec-future-reserved-words + // https://262.ecma-international.org/6.0/#sec-reserved-words + case 'true': + case 'false': + case 'null': + // https://262.ecma-international.org/6.0/#sec-keywords case 'await': case 'break': case 'case': @@ -79,7 +93,6 @@ bool invalidVariableName(String keyword, {bool strictMode = true}) { case 'import': case 'in': case 'instanceof': - case 'let': case 'new': case 'return': case 'super': @@ -99,6 +112,7 @@ bool invalidVariableName(String keyword, {bool strictMode = true}) { // http://www.ecma-international.org/ecma-262/6.0/#sec-identifiers-static-semantics-early-errors case 'implements': case 'interface': + case 'let': case 'package': case 'private': case 'protected': diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart new file mode 100644 index 0000000000..8d6a0a4116 --- /dev/null +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -0,0 +1,603 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ignore_for_file: constant_identifier_names + +import 'dart:async'; +import 'dart:collection'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'common.dart'; +import 'ddc_names.dart' as ddc_names; + +final _log = Logger('FrontendServerProxy'); +final dartaotruntimePath = p.join(sdkDir, 'bin', 'dartaotruntime'); +final frontendServerSnapshotPath = p.join( + sdkDir, + 'bin', + 'snapshots', + 'frontend_server_aot.dart.snapshot', +); + +/// A driver that proxies build requests to a [PersistentFrontendServer] +/// instance. +class FrontendServerProxyDriver { + PersistentFrontendServer? _frontendServer; + final _requestQueue = Queue<_CompilationRequest>(); + bool _isProcessing = false; + CompilerOutput? _cachedOutput; + + void init(PersistentFrontendServer frontendServer) { + _frontendServer = frontendServer; + } + + /// Sends a recompile request to the Frontend Server at [entrypoint] with + /// [invalidatedFiles]. + /// + /// The initial recompile request is treated as a full compile. + /// + /// [filesToWrite] contains JS files that should be written to the filesystem. + /// If left empty, all files are written. + Future recompileAndRecord( + String entrypoint, + List invalidatedFiles, + Iterable filesToWrite, + ) async { + final compilerOutput = await recompile(entrypoint, invalidatedFiles); + if (compilerOutput == null) { + throw Exception('Frontend Server failed to recompile $entrypoint'); + } + if (compilerOutput.errorCount != 0 || compilerOutput.errorMessage != null) { + throw Exception( + 'Frontend Server encountered errors during compilation: ' + '${compilerOutput.errorMessage}', + ); + } + _frontendServer!.recordFiles(); + for (final file in filesToWrite) { + _frontendServer!.writeFile(file); + } + return compilerOutput; + } + + Future compile(String entrypoint) async { + final completer = Completer(); + _requestQueue.add(_CompileRequest(entrypoint, completer)); + if (!_isProcessing) _processQueue(); + return completer.future; + } + + Future recompile( + String entrypoint, + List invalidatedFiles, + ) async { + final completer = Completer(); + _requestQueue.add( + _RecompileRequest(entrypoint, invalidatedFiles, completer), + ); + if (!_isProcessing) _processQueue(); + return completer.future; + } + + void _processQueue() async { + if (_isProcessing || _requestQueue.isEmpty) return; + + _isProcessing = true; + final request = _requestQueue.removeFirst(); + CompilerOutput? output; + try { + if (request is _CompileRequest) { + _cachedOutput = + output = await _frontendServer!.compile(request.entrypoint); + } else if (request is _RecompileRequest) { + // Compile the first [_RecompileRequest] as a [_CompileRequest] to warm + // up the Frontend Server. + if (_cachedOutput == null) { + output = + _cachedOutput = await _frontendServer!.compile( + request.entrypoint, + ); + } else { + output = await _frontendServer!.recompile( + request.entrypoint, + request.invalidatedFiles, + ); + } + } + if (output != null && output.errorCount == 0) { + _frontendServer!.accept(); + } else { + // We must await [reject]'s output, but we swallow the output since it + // doesn't provide useful information. + await _frontendServer!.reject(); + } + request.completer.complete(output); + } catch (e, s) { + request.completer.completeError(e, s); + } + + _isProcessing = false; + if (_requestQueue.isNotEmpty) { + _processQueue(); + } + } + + /// Clears the proxy driver's state between invocations of separate apps. + void reset() { + _requestQueue.clear(); + _isProcessing = false; + _cachedOutput = null; + _frontendServer?.reset(); + } + + Future terminate() async { + await _frontendServer!.shutdown(); + _frontendServer = null; + } +} + +abstract class _CompilationRequest { + final Completer completer; + _CompilationRequest(this.completer); +} + +class _CompileRequest extends _CompilationRequest { + final String entrypoint; + _CompileRequest(this.entrypoint, super.completer); +} + +class _RecompileRequest extends _CompilationRequest { + final String entrypoint; + final List invalidatedFiles; + _RecompileRequest(this.entrypoint, this.invalidatedFiles, super.completer); +} + +/// A single instance of the Frontend Server that persists across +/// compile/recompile requests. +class PersistentFrontendServer { + Process? _server; + final StdoutHandler _stdoutHandler; + final StreamController _stdinController; + final Uri outputDillUri; + final WebMemoryFilesystem _fileSystem; + + PersistentFrontendServer._( + this._server, + this._stdoutHandler, + this._stdinController, + this.outputDillUri, + this._fileSystem, + ); + + static Future start({ + required String sdkRoot, + required Uri fileSystemRoot, + required Uri packagesFile, + }) async { + final outputDillUri = fileSystemRoot.resolve('output.dill'); + final platformDill = p.join(sdkDir, 'lib', '_internal', 'ddc_outline.dill'); + final args = [ + frontendServerSnapshotPath, + '--sdk-root=$sdkRoot', + '--incremental', + '--target=dartdevc', + '--dartdevc-module-format=ddc', + '--dartdevc-canary', + '--no-js-strongly-connected-components', + '--packages=${packagesFile.toFilePath()}', + '--experimental-emit-debug-metadata', + '--filesystem-scheme=$multiRootScheme', + '--filesystem-root=${fileSystemRoot.toFilePath()}', + '--platform=$platformDill', + '--output-dill=${outputDillUri.toFilePath()}', + '--output-incremental-dill=${outputDillUri.toFilePath()}', + ]; + final process = await Process.start(dartaotruntimePath, args); + final fileSystem = WebMemoryFilesystem(fileSystemRoot); + final stdoutHandler = StdoutHandler(logger: _log); + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen(stdoutHandler.handler); + process.stderr.transform(utf8.decoder).listen(stderr.writeln); + + final stdinController = StreamController(); + stdinController.stream.listen(process.stdin.writeln); + + return PersistentFrontendServer._( + process, + stdoutHandler, + stdinController, + outputDillUri, + fileSystem, + ); + } + + Future compile(String entrypoint) async { + _stdoutHandler.reset(); + _stdinController.add('compile $entrypoint'); + return await _stdoutHandler.compilerOutput!.future; + } + + /// Either [accept] or [reject] should be called after every [recompile] call. + Future recompile( + String entrypoint, + List invalidatedFiles, + ) async { + _stdoutHandler.reset(); + final inputKey = const Uuid().v4(); + _stdinController.add('recompile $entrypoint $inputKey'); + for (final file in invalidatedFiles) { + _stdinController.add(file.toString()); + } + _stdinController.add(inputKey); + return await _stdoutHandler.compilerOutput!.future; + } + + void accept() { + _stdinController.add('accept'); + } + + Future reject() async { + _stdoutHandler.reset(expectSources: false); + _stdinController.add('reject'); + return await _stdoutHandler.compilerOutput!.future; + } + + void reset() { + _stdoutHandler.reset(); + _stdinController.add('reset'); + } + + /// Records all modified files into the in-memory filesystem. + void recordFiles() { + final outputDillPath = outputDillUri.toFilePath(); + final codeFile = File('$outputDillPath.sources'); + final manifestFile = File('$outputDillPath.json'); + final sourcemapFile = File('$outputDillPath.map'); + final metadataFile = File('$outputDillPath.metadata'); + _fileSystem.update(codeFile, manifestFile, sourcemapFile, metadataFile); + } + + void writeCompilerOutput(CompilerOutput output) { + if (output.errorCount != 0 || output.errorMessage != null) { + throw StateError('Attempting to write compiler output with errors.'); + } + _fileSystem.writeToDisk(_fileSystem.jsRootUri); + } + + void writeFile(String fileName) { + _fileSystem.writeFileToDisk(_fileSystem.jsRootUri, fileName); + } + + Future shutdown() async { + _stdinController.add('quit'); + await _server?.exitCode; + await _stdinController.close(); + _server = null; + } +} + +class CompilerOutput { + const CompilerOutput( + this.outputFilename, + this.errorCount, + this.sources, { + this.expressionData, + this.errorMessage, + }); + + final String outputFilename; + final int errorCount; + final List sources; + + /// Non-null for expression compilation requests. + final Uint8List? expressionData; + + /// Non-null when a compilation error was encountered. + final String? errorMessage; +} + +// A simple in-memory filesystem for handling the Frontend Server's compiled +// output. +class WebMemoryFilesystem { + /// The root directory's URI from which JS file are being served. + final Uri jsRootUri; + + final Map files = {}; + final Map sourcemaps = {}; + final Map metadata = {}; + + final List libraries = []; + + WebMemoryFilesystem(this.jsRootUri); + + /// Clears all files registered in the filesystem. + void clearWritableState() { + files.clear(); + sourcemaps.clear(); + metadata.clear(); + } + + /// Writes the entirety of this filesystem to [outputDirectoryUri]. + void writeToDisk(Uri outputDirectoryUri) { + assert( + Directory.fromUri(outputDirectoryUri).existsSync(), + '$outputDirectoryUri does not exist.', + ); + final filesToWrite = {...files, ...sourcemaps, ...metadata}; + _writeToDisk(outputDirectoryUri, filesToWrite); + } + + /// Writes [fileName] and its associated sourcemap and metadata files to + /// [outputDirectoryUri]. + void writeFileToDisk(Uri outputDirectoryUri, String fileName) { + assert( + Directory.fromUri(outputDirectoryUri).existsSync(), + '$outputDirectoryUri does not exist.', + ); + var sourceFile = fileName; + if (sourceFile.startsWith('package:')) { + sourceFile = + 'packages/${sourceFile.substring('package:'.length, sourceFile.length)}'; + } else if (sourceFile.startsWith('$multiRootScheme:///')) { + sourceFile = sourceFile.substring( + '$multiRootScheme:///'.length, + sourceFile.length, + ); + } + final sourceMapFile = '$sourceFile.map'; + final metadataFile = '$sourceFile.metadata'; + final filesToWrite = { + sourceFile: files[sourceFile]!, + sourceMapFile: sourcemaps[sourceMapFile]!, + metadataFile: metadata[metadataFile]!, + }; + + _writeToDisk(outputDirectoryUri, filesToWrite); + } + + /// Writes [rawFilesToWrite] to [outputDirectoryUri]. + void _writeToDisk( + Uri outputDirectoryUri, + Map rawFilesToWrite, + ) { + rawFilesToWrite.forEach((path, content) { + final outputFileUri = outputDirectoryUri.resolve(path); + final outputFilePath = outputFileUri.toFilePath().replaceFirst( + '.dart.lib.js', + '.ddc.js', + ); + final outputFile = File(outputFilePath); + outputFile.createSync(recursive: true); + outputFile.writeAsBytesSync(content); + }); + } + + /// Update the filesystem with the provided source and manifest files. + /// + /// Returns the list of updated files. + List update( + File codeFile, + File manifestFile, + File sourcemapFile, + File metadataFile, + ) { + final updatedFiles = []; + final codeBytes = codeFile.readAsBytesSync(); + final sourcemapBytes = sourcemapFile.readAsBytesSync(); + final manifest = Map.castFrom( + json.decode(manifestFile.readAsStringSync()) as Map, + ); + final metadataBytes = metadataFile.readAsBytesSync(); + + for (final filePath in manifest.keys) { + final fileUri = Uri.file(filePath); + final Map offsets = + Map.castFrom( + manifest[filePath] as Map, + ); + final codeOffsets = (offsets['code'] as List).cast(); + final sourcemapOffsets = + (offsets['sourcemap'] as List).cast(); + final metadataOffsets = + (offsets['metadata'] as List).cast(); + + if (codeOffsets.length != 2 || + sourcemapOffsets.length != 2 || + metadataOffsets.length != 2) { + _log.severe('Invalid manifest byte offsets: $offsets'); + continue; + } + + final codeStart = codeOffsets[0]; + final codeEnd = codeOffsets[1]; + if (codeStart < 0 || codeEnd > codeBytes.lengthInBytes) { + _log.severe('Invalid byte index: [$codeStart, $codeEnd]'); + continue; + } + + final byteView = Uint8List.view( + codeBytes.buffer, + codeStart, + codeEnd - codeStart, + ); + final fileName = + filePath.startsWith('/') ? filePath.substring(1) : filePath; + files[fileName] = byteView; + final moduleName = ddc_names.libraryUriToJsIdentifier(fileUri); + // TODO(markzipan): This is an overly simple heuristic to resolve the + // original Dart file. Replace this if it no longer holds. + var dartFileName = fileName; + if (dartFileName.endsWith('.lib.js')) { + dartFileName = fileName.substring( + 0, + fileName.length - '.lib.js'.length, + ); + } + final fullyResolvedFileUri = jsRootUri.resolve(fileName); + // TODO(markzipan): This is a simple hack to resolve kernel library URIs + // from JS files and might not generalize. + var libraryName = dartFileName; + if (libraryName.startsWith('packages/')) { + final libraryNameWithoutPrefix = libraryName.substring( + 'packages/'.length, + libraryName.length, + ); + libraryName = 'package:$libraryNameWithoutPrefix'; + } else { + libraryName = '$multiRootScheme:///$libraryName'; + } + final libraryInfo = LibraryInfo( + moduleName: moduleName, + libraryName: libraryName, + dartSourcePath: dartFileName, + jsSourcePath: fullyResolvedFileUri.toFilePath(), + ); + libraries.add(libraryInfo); + updatedFiles.add(fileName); + + final sourcemapStart = sourcemapOffsets[0]; + final sourcemapEnd = sourcemapOffsets[1]; + if (sourcemapStart < 0 || sourcemapEnd > sourcemapBytes.lengthInBytes) { + continue; + } + final sourcemapView = Uint8List.view( + sourcemapBytes.buffer, + sourcemapStart, + sourcemapEnd - sourcemapStart, + ); + final sourcemapName = '$fileName.map'; + sourcemaps[sourcemapName] = sourcemapView; + + final metadataStart = metadataOffsets[0]; + final metadataEnd = metadataOffsets[1]; + if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) { + _log.severe('Invalid byte index: [$metadataStart, $metadataEnd]'); + continue; + } + final metadataView = Uint8List.view( + metadataBytes.buffer, + metadataStart, + metadataEnd - metadataStart, + ); + metadata['$fileName.metadata'] = metadataView; + } + return updatedFiles; + } +} + +/// Bundles information associated with a DDC library. +class LibraryInfo { + final String moduleName; + final String libraryName; + final String dartSourcePath; + final String jsSourcePath; + + LibraryInfo({ + required this.moduleName, + required this.libraryName, + required this.dartSourcePath, + required this.jsSourcePath, + }); + + @override + String toString() => + 'LibraryInfo($moduleName, $libraryName, $dartSourcePath, $jsSourcePath)'; +} + +enum StdoutState { CollectDiagnostic, CollectDependencies } + +/// Handles stdin/stdout communication with the Frontend Server. +class StdoutHandler { + StdoutHandler({required Logger logger}) : _logger = logger { + reset(); + } + final Logger _logger; + + String? boundaryKey; + StdoutState state = StdoutState.CollectDiagnostic; + Completer? compilerOutput; + final sources = []; + + var _suppressCompilerMessages = false; + var _expectSources = true; + var _errorBuffer = StringBuffer(); + + void handler(String message) { + const kResultPrefix = 'result '; + if (boundaryKey == null && message.startsWith(kResultPrefix)) { + boundaryKey = message.substring(kResultPrefix.length); + return; + } + final messageBoundaryKey = boundaryKey; + if (messageBoundaryKey != null && message.startsWith(messageBoundaryKey)) { + if (_expectSources) { + if (state == StdoutState.CollectDiagnostic) { + state = StdoutState.CollectDependencies; + return; + } + } + if (message.length <= messageBoundaryKey.length) { + compilerOutput?.complete(); + return; + } + final spaceDelimiter = message.lastIndexOf(' '); + final fileName = message.substring( + messageBoundaryKey.length + 1, + spaceDelimiter, + ); + final errorCount = int.parse( + message.substring(spaceDelimiter + 1).trim(), + ); + + final output = CompilerOutput( + fileName, + errorCount, + sources, + expressionData: null, + errorMessage: _errorBuffer.isNotEmpty ? _errorBuffer.toString() : null, + ); + compilerOutput?.complete(output); + return; + } + switch (state) { + case StdoutState.CollectDiagnostic when _suppressCompilerMessages: + _logger.info(message); + _errorBuffer.writeln(message); + case StdoutState.CollectDiagnostic: + _logger.warning(message); + _errorBuffer.writeln(message); + case StdoutState.CollectDependencies: + switch (message[0]) { + case '+': + sources.add(Uri.parse(message.substring(1))); + case '-': + sources.remove(Uri.parse(message.substring(1))); + default: + _logger.warning('Ignoring unexpected prefix for $message uri'); + } + } + } + + // This is needed to get ready to process next compilation result output, + // with its own boundary key and new completer. + void reset({ + bool suppressCompilerMessages = false, + bool expectSources = true, + }) { + boundaryKey = null; + compilerOutput = Completer(); + _suppressCompilerMessages = suppressCompilerMessages; + _expectSources = expectSources; + state = StdoutState.CollectDiagnostic; + _errorBuffer = StringBuffer(); + } +} diff --git a/build_modules/lib/src/kernel_builder.dart b/build_modules/lib/src/kernel_builder.dart index 80644ecc88..222bb5bddb 100644 --- a/build_modules/lib/src/kernel_builder.dart +++ b/build_modules/lib/src/kernel_builder.dart @@ -17,12 +17,11 @@ import 'package:scratch_space/scratch_space.dart'; import 'package:stream_transform/stream_transform.dart'; import '../build_modules.dart'; +import 'common.dart'; import 'errors.dart'; import 'module_cache.dart'; import 'workers.dart'; -const multiRootScheme = 'org-dartlang-app'; - /// A builder which can output kernel files for a given sdk. /// /// This creates kernel files based on [moduleExtension] files, which are what @@ -431,7 +430,7 @@ Future _addRequestArguments( request.arguments.addAll([ '--dart-sdk-summary=${Uri.file(p.join(sdkDir, sdkKernelPath))}', '--output=${outputFile.path}', - '--packages-file=$multiRootScheme:///${p.join('.dart_tool', 'package_config.json')}', + '--packages-file=$multiRootScheme:///$packagesFilePath', '--multi-root-scheme=$multiRootScheme', '--exclude-non-sources', summaryOnly ? '--summary-only' : '--no-summary-only', diff --git a/build_modules/lib/src/module_builder.dart b/build_modules/lib/src/module_builder.dart index 85a621b5b6..f421ff3136 100644 --- a/build_modules/lib/src/module_builder.dart +++ b/build_modules/lib/src/module_builder.dart @@ -7,12 +7,14 @@ import 'dart:async'; import 'package:build/build.dart'; import 'package:collection/collection.dart'; +import 'meta_module_builder.dart'; import 'meta_module_clean_builder.dart'; import 'module_cache.dart'; import 'module_library.dart'; import 'module_library_builder.dart' show moduleLibraryExtension; import 'modules.dart'; import 'platform.dart'; +import 'scratch_space.dart'; /// The extension for serialized module assets. String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; @@ -22,7 +24,15 @@ String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; class ModuleBuilder implements Builder { final DartPlatform _platform; - ModuleBuilder(this._platform) + /// Emits DDC code with the Library Bundle module system, which supports hot + /// reload. + /// + /// If set, this builder will consume raw meta modules (instead of clean). + /// Clean meta modules are only used for DDC's AMD module system due its + /// requirement that self-referential libraries be bundled. + final bool usesWebHotReload; + + ModuleBuilder(this._platform, {this.usesWebHotReload = false}) : buildExtensions = { '.dart': [moduleExtension(_platform)], }; @@ -32,31 +42,31 @@ class ModuleBuilder implements Builder { @override Future build(BuildStep buildStep) async { - final cleanMetaModules = await buildStep.fetchResource(metaModuleCache); + final metaModules = await buildStep.fetchResource(metaModuleCache); + final metaModuleExtensionString = + usesWebHotReload + ? metaModuleExtension(_platform) + : metaModuleCleanExtension(_platform); final metaModule = - (await cleanMetaModules.find( - AssetId( - buildStep.inputId.package, - 'lib/${metaModuleCleanExtension(_platform)}', - ), + (await metaModules.find( + AssetId(buildStep.inputId.package, 'lib/$metaModuleExtensionString'), buildStep, ))!; var outputModule = metaModule.modules.firstWhereOrNull( (m) => m.primarySource == buildStep.inputId, ); - if (outputModule == null) { - final serializedLibrary = await buildStep.readAsString( - buildStep.inputId.changeExtension(moduleLibraryExtension), - ); - final libraryModule = ModuleLibrary.deserialize( - buildStep.inputId, - serializedLibrary, + final serializedLibrary = await buildStep.readAsString( + buildStep.inputId.changeExtension(moduleLibraryExtension), + ); + final libraryModule = ModuleLibrary.deserialize( + buildStep.inputId, + serializedLibrary, + ); + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + if (outputModule == null && libraryModule.hasMain) { + outputModule = metaModule.modules.firstWhere( + (m) => m.sources.contains(buildStep.inputId), ); - if (libraryModule.hasMain) { - outputModule = metaModule.modules.firstWhere( - (m) => m.sources.contains(buildStep.inputId), - ); - } } if (outputModule == null) return; final modules = await buildStep.fetchResource(moduleCache); @@ -65,5 +75,13 @@ class ModuleBuilder implements Builder { buildStep, outputModule, ); + if (usesWebHotReload) { + // All sources must be declared before the Frontend Server is invoked, as + // it only accepts the main entrypoint as its compilation target. + await buildStep.trackStage( + 'EnsureAssets', + () => scratchSpace.ensureAssets(outputModule!.sources, buildStep), + ); + } } } diff --git a/build_modules/lib/src/modules.dart b/build_modules/lib/src/modules.dart index 59163ec069..a7e86449e7 100644 --- a/build_modules/lib/src/modules.dart +++ b/build_modules/lib/src/modules.dart @@ -148,6 +148,7 @@ class Module { Future> computeTransitiveDependencies( BuildStep buildStep, { bool throwIfUnsupported = false, + bool computeStronglyConnectedComponents = true, }) async { final modules = await buildStep.fetchResource(moduleCache); final transitiveDeps = {}; @@ -183,13 +184,67 @@ class Module { if (throwIfUnsupported && unsupportedModules.isNotEmpty) { throw UnsupportedModules(unsupportedModules); } - final orderedModules = stronglyConnectedComponents( - transitiveDeps.values, - (m) => m.directDependencies.map((s) => transitiveDeps[s]!), - equals: (a, b) => a.primarySource == b.primarySource, - hashCode: (m) => m.primarySource.hashCode, - ); - return orderedModules.map((c) => c.single).toList(); + + if (computeStronglyConnectedComponents) { + final orderedModules = stronglyConnectedComponents( + transitiveDeps.values, + (m) => m.directDependencies.map((s) => transitiveDeps[s]!), + equals: (a, b) => a.primarySource == b.primarySource, + hashCode: (m) => m.primarySource.hashCode, + ); + return orderedModules.map((c) => c.single).toList(); + } + return transitiveDeps.values.toList(); + } + + /// Returns all [AssetId]s in the transitive dependencies of this module in + /// no specific order. + /// + /// Throws a [MissingModulesException] if there are any missing modules. This + /// typically means that somebody is trying to import a non-existing file. + /// + /// If [throwIfUnsupported] is `true`, then an [UnsupportedModules] + /// will be thrown if there are any modules that are not supported. + Future> computeTransitiveAssets( + BuildStep buildStep, { + bool throwIfUnsupported = false, + }) async { + final modules = await buildStep.fetchResource(moduleCache); + final transitiveDeps = {}; + final modulesToCrawl = {primarySource}; + final missingModuleSources = {}; + final unsupportedModules = {}; + final seenSources = {}; + + while (modulesToCrawl.isNotEmpty) { + final next = modulesToCrawl.last; + modulesToCrawl.remove(next); + if (transitiveDeps.containsKey(next)) continue; + final nextModuleId = next.changeExtension(moduleExtension(platform)); + final module = await modules.find(nextModuleId, buildStep); + if (module == null || module.isMissing) { + missingModuleSources.add(next); + continue; + } + if (throwIfUnsupported && !module.isSupported) { + unsupportedModules.add(module); + } + transitiveDeps[next] = module; + modulesToCrawl.addAll(module.directDependencies); + seenSources.addAll(module.sources); + } + + if (missingModuleSources.isNotEmpty) { + throw await MissingModulesException.create( + missingModuleSources, + transitiveDeps.values.toList()..add(this), + buildStep, + ); + } + if (throwIfUnsupported && unsupportedModules.isNotEmpty) { + throw UnsupportedModules(unsupportedModules); + } + return seenSources; } } diff --git a/build_modules/lib/src/scratch_space.dart b/build_modules/lib/src/scratch_space.dart index 9ef56e5bfe..6c8bf35f32 100644 --- a/build_modules/lib/src/scratch_space.dart +++ b/build_modules/lib/src/scratch_space.dart @@ -44,11 +44,15 @@ final scratchSpaceResource = Resource( } return scratchSpace; }, + dispose: (scratchSpace) { + scratchSpace.dispose(); + }, beforeExit: () async { // The workers are running inside the scratch space, so wait for them to // shut down before deleting it. await dartdevkWorkersAreDone; await frontendWorkersAreDone; + await frontendServerProxyWorkersAreDone; // Attempt to clean up the scratch space. Even after waiting for the workers // to shut down we might get file system exceptions on windows for an // arbitrary amount of time, so do retries with an exponential backoff. @@ -83,7 +87,7 @@ final scratchSpaceResource = Resource( }, ); -/// Modifies all package uris in [rootConfig] to work with the sctrach_space +/// Modifies all package uris in [rootConfig] to work with the scratch_space /// layout. These are uris of the form `../packages/`. /// /// Also modifies the `packageUri` for each package to be empty since the @@ -109,12 +113,12 @@ String _scratchSpacePackageConfig(String rootConfig, Uri packageConfigUri) { rootUri = rootUri.replace(path: '${rootUri.path}/'); } // We expect to see exactly one package where the root uri is equal to - // the current directory, and that is the current packge. + // the current directory, and that is the current package. if (rootUri == _currentDirUri) { assert(!foundRoot); foundRoot = true; - package['rootUri'] = '../'; - package['packageUri'] = '../packages/${package['name']}/'; + package['packageUri'] = ''; + package['rootUri'] = '../packages/${package['name']}/'; } else { package['rootUri'] = '../packages/${package['name']}/'; package.remove('packageUri'); diff --git a/build_modules/lib/src/workers.dart b/build_modules/lib/src/workers.dart index ba99eaf1d3..6278c13391 100644 --- a/build_modules/lib/src/workers.dart +++ b/build_modules/lib/src/workers.dart @@ -10,10 +10,10 @@ import 'package:bazel_worker/driver.dart'; import 'package:build/build.dart'; import 'package:path/path.dart' as p; +import 'common.dart'; +import 'frontend_server_driver.dart'; import 'scratch_space.dart'; -final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); - // If no terminal is attached, prevent a new one from launching. final _processMode = stdin.hasTerminal @@ -108,3 +108,43 @@ final frontendDriverResource = Resource( __frontendDriver = null; }, ); + +/// Completes once the Frontend Service proxy workers have been shut down. +Future get frontendServerProxyWorkersAreDone => + _frontendServerProxyWorkersAreDoneCompleter?.future ?? Future.value(); +Completer? _frontendServerProxyWorkersAreDoneCompleter; + +FrontendServerProxyDriver get _frontendServerProxyDriver { + _frontendServerProxyWorkersAreDoneCompleter ??= Completer(); + return __frontendServerProxyDriver ??= FrontendServerProxyDriver(); +} + +FrontendServerProxyDriver? __frontendServerProxyDriver; + +/// Manages a shared set of workers that proxy requests to a single +/// [persistentFrontendServerResource]. +final frontendServerProxyDriverResource = Resource( + () async => _frontendServerProxyDriver, + beforeExit: () async { + _frontendServerProxyWorkersAreDoneCompleter?.complete(); + await __frontendServerProxyDriver?.terminate(); + _frontendServerProxyWorkersAreDoneCompleter = null; + __frontendServerProxyDriver = null; + }, +); + +PersistentFrontendServer? __persistentFrontendServer; + +/// Manages a single persistent instance of the Frontend Server targeting DDC. +final persistentFrontendServerResource = Resource( + () async => + __persistentFrontendServer ??= await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: scratchSpace.tempDir.uri, + packagesFile: scratchSpace.tempDir.uri.resolve(packagesFilePath), + ), + beforeExit: () async { + await __persistentFrontendServer?.shutdown(); + __persistentFrontendServer = null; + }, +); diff --git a/build_modules/pubspec.yaml b/build_modules/pubspec.yaml index 2bb7fffdd8..8fc2085501 100644 --- a/build_modules/pubspec.yaml +++ b/build_modules/pubspec.yaml @@ -16,6 +16,7 @@ dependencies: build: '>=2.0.0 <5.0.0' collection: ^1.15.0 crypto: ^3.0.0 + file: ^7.0.1 glob: ^2.0.0 graphs: ^2.0.0 json_annotation: ^4.3.0 @@ -23,6 +24,7 @@ dependencies: path: ^1.8.0 scratch_space: ^1.0.0 stream_transform: ^2.0.0 + uuid: ^4.4.2 dev_dependencies: a: diff --git a/build_modules/test/frontend_server_driver_test.dart b/build_modules/test/frontend_server_driver_test.dart new file mode 100644 index 0000000000..7e37d0ee83 --- /dev/null +++ b/build_modules/test/frontend_server_driver_test.dart @@ -0,0 +1,179 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:build_modules/src/common.dart'; +import 'package:build_modules/src/frontend_server_driver.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + group('Frontend Server', () { + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + }); + + tearDown(() async { + await server.shutdown(); + await tempDir.delete(recursive: true); + }); + + test('can compile a simple dart file', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + expect(output.outputFilename, endsWith('output.dill')); + expect(output.sources, contains(entrypoint)); + server.accept(); + }); + + test('can handle compilation errors', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, isNotNull); + expect( + output.errorMessage, + contains("Error: Can't find '}' to match '{'"), + ); + await server.reject(); + }); + + test('can reject and recompile with an invalidated file', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + final dep = tempDir.uri.resolve('dep.dart'); + File( + entrypoint.toFilePath(), + ).writeAsStringSync("import 'dep.dart'; void main() {}"); + File(dep.toFilePath()).writeAsStringSync('// empty'); + + var output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + + File(dep.toFilePath()).writeAsStringSync('invalid dart code'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, contains("Error: Expected ';' after this.")); + await server.reject(); + + File(dep.toFilePath()).writeAsStringSync('// empty'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + }); + }); + + group('Frontend Server Proxy', () { + late FrontendServerProxyDriver driver; + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + driver = FrontendServerProxyDriver(); + driver.init(server); + }); + + tearDown(() async { + await driver.terminate(); + await tempDir.delete(recursive: true); + }); + + test('can recompile after valid edits', () async { + final file1 = tempDir.uri.resolve('file1.dart'); + File(file1.toFilePath()).writeAsStringSync('void main() {}'); + final file2 = tempDir.uri.resolve('file2.dart'); + File(file2.toFilePath()).writeAsStringSync('void main() {}'); + + var future1 = driver.recompile(file1.toString(), []); + var future2 = driver.recompile(file2.toString(), []); + + var results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + + File( + file1.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + File( + file2.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + + future1 = driver.recompile(file1.toString(), [file1]); + future2 = driver.recompile(file2.toString(), [file2]); + + results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + }); + + test('recompileAndRecord successfully records and writes files', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final jsFESOutputPath = p.join(tempDir.path, 'entrypoint.dart.lib.js'); + final output = await driver.recompileAndRecord( + '$multiRootScheme:///entrypoint.dart', + [entrypoint], + ['entrypoint.dart.lib.js'], + ); + + expect(output, isNotNull); + expect(output.errorCount, 0); + + final jsOutputFile = File( + jsFESOutputPath.replaceFirst('.dart.lib.js', '.ddc.js'), + ); + expect(jsOutputFile.existsSync(), isTrue); + final content = jsOutputFile.readAsStringSync(); + expect(content, contains('function main()')); + }); + }); +} diff --git a/build_modules/test/meta_module_builder_test.dart b/build_modules/test/meta_module_builder_test.dart index 56b11dc779..c208399ee6 100644 --- a/build_modules/test/meta_module_builder_test.dart +++ b/build_modules/test/meta_module_builder_test.dart @@ -5,7 +5,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_modules/test/meta_module_test.dart b/build_modules/test/meta_module_test.dart index 1917d48e54..5b0aa9ce34 100644 --- a/build_modules/test/meta_module_test.dart +++ b/build_modules/test/meta_module_test.dart @@ -8,7 +8,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/common.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_modules/test/module_builder_test.dart b/build_modules/test/module_builder_test.dart index 065ffd9b05..3a684b5cfe 100644 --- a/build_modules/test/module_builder_test.dart +++ b/build_modules/test/module_builder_test.dart @@ -7,7 +7,6 @@ import 'dart:convert'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; @@ -49,8 +48,14 @@ void main() { 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( metaModule.toJson(), ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), 'a|lib/c$moduleLibraryExtension': ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), 'a|lib/e$moduleLibraryExtension': ModuleLibrary.fromSource(assetE, '').serialize(), }, @@ -61,4 +66,67 @@ void main() { }, ); }); + + test('can serialize modules and output all modules when strongly connected ' + 'components are disabled', () async { + final assetA = AssetId('a', 'lib/a.dart'); + final assetB = AssetId('a', 'lib/b.dart'); + final assetC = AssetId('a', 'lib/c.dart'); + final assetD = AssetId('a', 'lib/d.dart'); + final assetE = AssetId('a', 'lib/e.dart'); + final moduleA = Module(assetA, [assetA], [], platform, true); + final moduleB = Module( + assetB, + [assetB, assetC], + [], + platform, + true, + ); + final moduleC = Module(assetC, [assetC], [], platform, true); + final moduleD = Module( + assetD, + [assetD, assetE], + [], + platform, + false, + ); + final moduleE = Module(assetE, [assetE], [], platform, true); + final metaModule = MetaModule([ + moduleA, + moduleB, + moduleC, + moduleD, + moduleE, + ]); + await testBuilder( + ModuleBuilder(platform, usesWebHotReload: true), + { + 'a|lib/a.dart': '', + 'a|lib/b.dart': '', + 'a|lib/c.dart': '', + 'a|lib/d.dart': '', + 'a|lib/e.dart': '', + 'a|lib/${metaModuleExtension(platform)}': jsonEncode( + metaModule.toJson(), + ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), + 'a|lib/c$moduleLibraryExtension': + ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), + 'a|lib/e$moduleLibraryExtension': + ModuleLibrary.fromSource(assetE, '').serialize(), + }, + outputs: { + 'a|lib/a${moduleExtension(platform)}': encodedMatchesModule(moduleA), + 'a|lib/b${moduleExtension(platform)}': encodedMatchesModule(moduleB), + 'a|lib/c${moduleExtension(platform)}': encodedMatchesModule(moduleC), + 'a|lib/d${moduleExtension(platform)}': encodedMatchesModule(moduleD), + 'a|lib/e${moduleExtension(platform)}': encodedMatchesModule(moduleE), + }, + ); + }); } diff --git a/build_runner/lib/src/constants.dart b/build_runner/lib/src/constants.dart index 67d9d261a6..39b5123f04 100644 --- a/build_runner/lib/src/constants.dart +++ b/build_runner/lib/src/constants.dart @@ -19,6 +19,9 @@ final String assetGraphPath = '$cacheDirectoryPath/asset_graph.json'; /// The directory to which hidden assets will be written. String get generatedOutputDirectory => '$cacheDirectoryPath/generated'; +/// Relative path to the cache directory from the root package dir. +const cacheDir = '.dart_tool/build'; + /// The dart binary from the current sdk. final dartBinary = p.join(sdkBin, 'dart'); diff --git a/build_test/lib/src/internal_test_reader_writer.dart b/build_test/lib/src/internal_test_reader_writer.dart index a1fb1505e3..87de6a8ed8 100644 --- a/build_test/lib/src/internal_test_reader_writer.dart +++ b/build_test/lib/src/internal_test_reader_writer.dart @@ -274,4 +274,10 @@ class _ReaderWriterTestingImpl implements ReaderWriterTesting { void delete(AssetId id) => _readerWriter.filesystem.deleteSync( _readerWriter.assetPathProvider.pathFor(id), ); + + @override + void reset() { + _readerWriter.assetsRead.clear(); + _readerWriter.assetsWritten.clear(); + } } diff --git a/build_test/lib/src/test_builder.dart b/build_test/lib/src/test_builder.dart index 1ff44e0c41..4a51075558 100644 --- a/build_test/lib/src/test_builder.dart +++ b/build_test/lib/src/test_builder.dart @@ -9,6 +9,8 @@ import 'package:build/build.dart'; import 'package:build/experiments.dart'; import 'package:build_config/build_config.dart'; // ignore: implementation_imports +import 'package:build_runner/src/build/asset_graph/graph.dart'; +// ignore: implementation_imports import 'package:build_runner/src/internal.dart'; import 'package:built_collection/built_collection.dart'; import 'package:glob/glob.dart'; @@ -137,8 +139,10 @@ Future testBuilder( PackageConfig? packageConfig, Resolvers? resolvers, TestReaderWriter? readerWriter, + AssetGraph? assetGraph, bool verbose = false, bool enableLowResourceMode = false, + bool performCleanup = true, }) async { return testBuilders( [builder], @@ -152,8 +156,10 @@ Future testBuilder( packageConfig: packageConfig, resolvers: resolvers, readerWriter: readerWriter, + assetGraph: assetGraph, verbose: verbose, enableLowResourceMode: enableLowResourceMode, + performCleanup: performCleanup, ); } @@ -184,8 +190,10 @@ Future testBuilders( Map> appliesBuilders = const {}, bool testingBuilderConfig = true, TestReaderWriter? readerWriter, + AssetGraph? assetGraph, bool verbose = false, bool enableLowResourceMode = false, + bool performCleanup = true, }) { final builderFactories = []; final optionalBuilderFactories = Set.identity(); @@ -227,6 +235,7 @@ Future testBuilders( readerWriter: readerWriter, verbose: verbose, enableLowResourceMode: enableLowResourceMode, + performCleanup: performCleanup, ); } @@ -313,8 +322,10 @@ Future testBuilderFactories( Map> appliesBuilders = const {}, bool testingBuilderConfig = true, TestReaderWriter? readerWriter, + AssetGraph? assetGraph, bool verbose = false, bool enableLowResourceMode = false, + bool performCleanup = true, }) async { onLog ??= _printOnFailureOrWrite; @@ -344,16 +355,18 @@ Future testBuilderFactories( } rootPackage ??= allPackages.first; - readerWriter ??= TestReaderWriter(rootPackage: rootPackage); - - sourceAssets.forEach((serializedId, contents) { - final id = makeAssetId(serializedId); - if (contents is String) { - readerWriter!.testing.writeString(id, contents); - } else if (contents is List) { - readerWriter!.testing.writeBytes(id, contents); - } - }); + // Only write source assets if this is a new build (non-incremental). + if (readerWriter == null) { + readerWriter = TestReaderWriter(rootPackage: rootPackage); + sourceAssets.forEach((serializedId, contents) { + final id = makeAssetId(serializedId); + if (contents is String) { + readerWriter!.testing.writeString(id, contents); + } else if (contents is List) { + readerWriter!.testing.writeBytes(id, contents); + } + }); + } final inputFilter = isInput ?? generateFor?.contains ?? (_) => true; inputIds.retainWhere((id) => inputFilter('$id')); @@ -474,13 +487,17 @@ Future testBuilderFactories( ); await buildPlan.deleteFilesAndFolders(); + final previousAssetGraph = assetGraph ?? buildPlan.takePreviousAssetGraph(); + final buildSeries = BuildSeries(buildPlan); // Run the build. final buildResult = await buildSeries.run({}, recentlyBootstrapped: true); - // Do cleanup that would usually happen on process exit. - await buildSeries.close(); + if (performCleanup) { + // Do cleanup that would usually happen on process exit. + await buildSeries.close(); + } // Stop logging. buildLog.configuration = buildLog.configuration.rebuild((b) { @@ -492,6 +509,7 @@ Future testBuilderFactories( return TestBuilderResult( buildResult: buildResult, + assetGraph: previousAssetGraph, readerWriter: readerWriter, ); } @@ -500,6 +518,7 @@ class TestBuilderResult { @Deprecated('Use `succeeded`, `outputs` and `errors` instead.') final BuildResult buildResult; final TestReaderWriter readerWriter; + final AssetGraph? assetGraph; bool get succeeded => buildResult.status == BuildStatus.success; @@ -514,7 +533,11 @@ class TestBuilderResult { /// If the build failed there must be at least one error. BuiltList get errors => buildResult.errors; - TestBuilderResult({required this.buildResult, required this.readerWriter}); + TestBuilderResult({ + required this.buildResult, + required this.readerWriter, + this.assetGraph, + }); } void _printOnFailureOrWrite(LogRecord record) { diff --git a/build_test/lib/src/test_reader_writer.dart b/build_test/lib/src/test_reader_writer.dart index a9ddd000db..cdfe164564 100644 --- a/build_test/lib/src/test_reader_writer.dart +++ b/build_test/lib/src/test_reader_writer.dart @@ -92,4 +92,7 @@ abstract interface class ReaderWriterTesting { /// Deletes [id] from the [TestReaderWriter] in-memory filesystem. void delete(AssetId id); + + /// Resets state in this [TestReaderWriter] between rebuilds. + void reset(); } diff --git a/build_web_compilers/build.yaml b/build_web_compilers/build.yaml index 3f9c2eb887..52ed081832 100644 --- a/build_web_compilers/build.yaml +++ b/build_web_compilers/build.yaml @@ -1,6 +1,8 @@ targets: $default: builders: + build_web_compilers:entrypoint_marker: + enabled: true build_web_compilers:entrypoint: options: compiler: dart2js @@ -24,6 +26,7 @@ builders: - lib/src/dev_compiler/dart_sdk.js - lib/src/dev_compiler/dart_sdk.js.map - lib/src/dev_compiler/require.js + - lib/src/dev_compiler/ddc_module_loader.js is_optional: True auto_apply: none runs_before: ["build_web_compilers:entrypoint"] @@ -145,6 +148,20 @@ builders: compiler: dart2js applies_builders: - build_web_compilers:dart2js_archive_extractor + entrypoint_marker: + import: "package:build_web_compilers/builders.dart" + builder_factories: + - webEntrypointMarkerBuilder + build_extensions: + $web$: + - .web.entrypoint.json + required_inputs: + - .dart + build_to: cache + auto_apply: root_package + applies_builders: + - build_web_compilers:entrypoint + runs_before: ["build_web_compilers:ddc", "build_web_compilers:ddc_modules"] _stack_trace_mapper_copy: import: "tool/copy_builder.dart" builder_factories: diff --git a/build_web_compilers/lib/build_web_compilers.dart b/build_web_compilers/lib/build_web_compilers.dart index 92e101ead4..8044292333 100644 --- a/build_web_compilers/lib/build_web_compilers.dart +++ b/build_web_compilers/lib/build_web_compilers.dart @@ -3,15 +3,15 @@ // BSD-style license that can be found in the LICENSE file. export 'src/archive_extractor.dart' show Dart2JsArchiveExtractor; -export 'src/dev_compiler_builder.dart' +export 'src/common.dart' show - DevCompilerBuilder, fullKernelExtension, jsModuleErrorsExtension, jsModuleExtension, jsSourceMapExtension, metadataExtension, symbolsExtension; +export 'src/dev_compiler_builder.dart' show DevCompilerBuilder; export 'src/platforms.dart' show dart2jsPlatform, dart2wasmPlatform, ddcPlatform; export 'src/web_entrypoint_builder.dart' diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index cf8e66f207..ea74aeec0c 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -8,29 +8,54 @@ import 'package:collection/collection.dart'; import 'build_web_compilers.dart'; import 'src/common.dart'; +import 'src/ddc_frontend_server_builder.dart'; import 'src/sdk_js_compile_builder.dart'; import 'src/sdk_js_copy_builder.dart'; +import 'src/web_entrypoint_marker_builder.dart'; // Shared entrypoint builder -Builder webEntrypointBuilder(BuilderOptions options) => - WebEntrypointBuilder.fromOptions(options); +Builder webEntrypointBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return WebEntrypointBuilder.fromOptions(options); +} + +Builder webEntrypointMarkerBuilder(BuilderOptions options) => + WebEntrypointMarkerBuilder(); + +// DDC related builders +Builder ddcMetaModuleBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return MetaModuleBuilder.forOptions(ddcPlatform, options); +} -// Ddc related builders -Builder ddcMetaModuleBuilder(BuilderOptions options) => - MetaModuleBuilder.forOptions(ddcPlatform, options); -Builder ddcMetaModuleCleanBuilder(BuilderOptions _) => - MetaModuleCleanBuilder(ddcPlatform); -Builder ddcModuleBuilder(BuilderOptions _) => ModuleBuilder(ddcPlatform); +Builder ddcMetaModuleCleanBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return MetaModuleCleanBuilder(ddcPlatform); +} + +Builder ddcModuleBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return ModuleBuilder( + ddcPlatform, + usesWebHotReload: _readWebHotReloadOption(options), + ); +} Builder ddcBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); + _ensureSameDdcHotReloadOptions(options); _ensureSameDdcOptions(options); + if (_readWebHotReloadOption(options)) { + return DdcFrontendServerBuilder(); + } + return DevCompilerBuilder( useIncrementalCompiler: _readUseIncrementalCompilerOption(options), generateFullDill: _readGenerateFullDillOption(options), emitDebugSymbols: _readEmitDebugSymbolsOption(options), canaryFeatures: _readCanaryOption(options), + ddcModules: _readWebHotReloadOption(options), sdkKernelPath: sdkDdcKernelPath, trackUnusedInputs: _readTrackInputsCompilerOption(options), platform: ddcPlatform, @@ -42,6 +67,7 @@ final ddcKernelExtension = '.ddc.dill'; Builder ddcKernelBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); + _ensureSameDdcHotReloadOptions(options); _ensureSameDdcOptions(options); return KernelBuilder( @@ -55,11 +81,16 @@ Builder ddcKernelBuilder(BuilderOptions options) { } Builder sdkJsCopyRequirejs(BuilderOptions _) => SdkJsCopyBuilder(); -Builder sdkJsCompile(BuilderOptions options) => SdkJsCompileBuilder( - sdkKernelPath: 'lib/_internal/ddc_platform.dill', - outputPath: 'lib/src/dev_compiler/dart_sdk.js', - canaryFeatures: _readCanaryOption(options), -); +Builder sdkJsCompile(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return SdkJsCompileBuilder( + sdkKernelPath: 'lib/_internal/ddc_platform.dill', + outputPath: 'lib/src/dev_compiler/dart_sdk.js', + canaryFeatures: + _readWebHotReloadOption(options) || _readCanaryOption(options), + usesWebHotReload: _readWebHotReloadOption(options), + ); +} // Dart2js related builders Builder dart2jsMetaModuleBuilder(BuilderOptions options) => @@ -115,6 +146,23 @@ void _ensureSameDdcOptions(BuilderOptions options) { } } +void _ensureSameDdcHotReloadOptions(BuilderOptions options) { + final webHotReload = _readWebHotReloadOption(options); + if (_lastWebHotReloadValue != null) { + if (webHotReload != _lastWebHotReloadValue) { + throw ArgumentError( + '`web-hot-reload` must be configured the same across the following ' + 'builders: build_web_compilers:ddc, build_web_compilers|sdk_js, ' + 'build_web_compilers|entrypoint, and build_web_compilers|ddc_modules.' + '\n\nPlease use the `global_options` section in ' + '`build.yaml` or the `--define` flag to set global options.', + ); + } + } else { + _lastWebHotReloadValue = webHotReload; + } +} + bool _readUseIncrementalCompilerOption(BuilderOptions options) { return options.config[_useIncrementalCompilerOption] as bool? ?? true; } @@ -135,18 +183,24 @@ bool _readTrackInputsCompilerOption(BuilderOptions options) { return options.config[_trackUnusedInputsCompilerOption] as bool? ?? true; } +bool _readWebHotReloadOption(BuilderOptions options) { + return options.config[_webHotReloadOption] as bool? ?? false; +} + Map _readEnvironmentOption(BuilderOptions options) { final environment = options.config[_environmentOption] as Map? ?? const {}; return environment.map((key, value) => MapEntry('$key', '$value')); } Map? _previousDdcConfig; +bool? _lastWebHotReloadValue; const _useIncrementalCompilerOption = 'use-incremental-compiler'; const _generateFullDillOption = 'generate-full-dill'; const _emitDebugSymbolsOption = 'emit-debug-symbols'; const _canaryOption = 'canary'; const _trackUnusedInputsCompilerOption = 'track-unused-inputs'; const _environmentOption = 'environment'; +const _webHotReloadOption = 'web-hot-reload'; const _supportedOptions = [ _environmentOption, @@ -155,4 +209,5 @@ const _supportedOptions = [ _emitDebugSymbolsOption, _canaryOption, _trackUnusedInputsCompilerOption, + _webHotReloadOption, ]; diff --git a/build_web_compilers/lib/src/common.dart b/build_web_compilers/lib/src/common.dart index 2aaf3d9db9..1014e266fc 100644 --- a/build_web_compilers/lib/src/common.dart +++ b/build_web_compilers/lib/src/common.dart @@ -9,6 +9,14 @@ import 'package:build/build.dart'; import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; +final multiRootScheme = 'org-dartlang-app'; +final jsModuleErrorsExtension = '.ddc.js.errors'; +final jsModuleExtension = '.ddc.js'; +final jsSourceMapExtension = '.ddc.js.map'; +final metadataExtension = '.ddc.js.metadata'; +final symbolsExtension = '.ddc.js.symbols'; +final fullKernelExtension = '.ddc.full.dill'; + final defaultAnalysisOptionsId = AssetId( 'build_modules', 'lib/src/analysis_options.default.yaml', @@ -50,6 +58,15 @@ void validateOptions( } } +/// The url to compile for a source. +/// +/// Use the package: path for files under lib and the full absolute path for +/// other files. +String sourceArg(AssetId id) { + final uri = canonicalUriFor(id); + return uri.startsWith('package:') ? uri : '$multiRootScheme:///${id.path}'; +} + /// If [id] exists, assume it is a source map and fix up the source uris from /// it so they make sense in a browser context, then write the modified version /// using [writer]. @@ -85,3 +102,67 @@ Future fixAndCopySourceMap( await writer.writeAsString(id, jsonEncode(json)); } } + +void fixMetadataSources(Map json, Uri scratchUri) { + String updatePath(String path) => + Uri.parse(path).path.replaceAll(scratchUri.path, ''); + + final sourceMapUri = json['sourceMapUri'] as String?; + if (sourceMapUri != null) { + json['sourceMapUri'] = updatePath(sourceMapUri); + } + + final moduleUri = json['moduleUri'] as String?; + if (moduleUri != null) { + json['moduleUri'] = updatePath(moduleUri); + } + + final fullDillUri = json['fullDillUri'] as String?; + if (fullDillUri != null) { + json['fullDillUri'] = updatePath(fullDillUri); + } + + final libraries = json['libraries'] as List?; + if (libraries != null) { + for (final lib in libraries) { + final libraryJson = lib as Map?; + if (libraryJson != null) { + final fileUri = libraryJson['fileUri'] as String?; + if (fileUri != null) { + libraryJson['fileUri'] = updatePath(fileUri); + } + } + } + } +} + +/// The module name according to ddc for [jsId] which represents the real js +/// module file. +String ddcModuleName(AssetId jsId) { + final jsPath = + jsId.path.startsWith('lib/') + ? jsId.path.replaceFirst('lib/', 'packages/${jsId.package}/') + : jsId.path; + return jsPath.substring(0, jsPath.length - jsModuleExtension.length); +} + +String ddcLibraryId(AssetId jsId) { + final jsPath = + jsId.path.startsWith('lib/') + ? jsId.path.replaceFirst('lib/', 'package:${jsId.package}/') + : '$multiRootScheme://${jsId.path}'; + final prefix = jsPath.substring(0, jsPath.length - jsModuleExtension.length); + return '$prefix.dart'; +} + +AssetId changeAssetIdExtension( + AssetId inputId, + String inputExtension, + String outputExtension, +) { + assert(inputId.path.endsWith(inputExtension)); + final newPath = + inputId.path.substring(0, inputId.path.length - inputExtension.length) + + outputExtension; + return AssetId(inputId.package, newPath); +} diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart new file mode 100644 index 0000000000..6909d781a4 --- /dev/null +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -0,0 +1,116 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; + +import 'common.dart'; +import 'platforms.dart'; + +/// A builder that compiles DDC modules with the Frontend Server. +class DdcFrontendServerBuilder implements Builder { + DdcFrontendServerBuilder(); + + @override + final Map> buildExtensions = { + moduleExtension(ddcPlatform): [ + jsModuleExtension, + jsModuleErrorsExtension, + jsSourceMapExtension, + metadataExtension, + ], + }; + + @override + Future build(BuildStep buildStep) async { + final moduleContents = await buildStep.readAsString(buildStep.inputId); + final module = Module.fromJson( + json.decode(moduleContents) as Map, + ); + final ddcEntrypointId = module.primarySource; + // Entrypoints always have a `.module` file for ease of looking them up, + // but they might not be the primary source. + if (ddcEntrypointId.changeExtension(moduleExtension(ddcPlatform)) != + buildStep.inputId) { + return; + } + + Future handleError(Object e) async { + await buildStep.writeAsString( + ddcEntrypointId.changeExtension(jsModuleErrorsExtension), + '$e', + ); + log.severe('Error encountered: $e'); + } + + try { + await _compile(module, buildStep); + } catch (e) { + await handleError(e); + } + } + + /// Compile [module] with Frontend Server. + Future _compile(Module module, BuildStep buildStep) async { + final transitiveAssets = await buildStep.trackStage( + 'CollectTransitiveDeps', + () => module.computeTransitiveAssets(buildStep), + ); + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + final webEntrypointAsset = scratchSpace.entrypointAssetId; + await buildStep.trackStage( + 'EnsureAssets', + () => scratchSpace.ensureAssets([ + webEntrypointAsset, + ...transitiveAssets, + ], buildStep), + ); + final changedAssetUris = [ + for (final asset in scratchSpace.changedFilesInBuild) asset.uri, + ]; + final ddcEntrypointId = module.primarySource; + final jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); + final jsFESOutputId = ddcEntrypointId.changeExtension('.dart.lib.js'); + + final frontendServer = await buildStep.fetchResource( + persistentFrontendServerResource, + ); + final driver = await buildStep.fetchResource( + frontendServerProxyDriverResource, + ); + driver.init(frontendServer); + + // Request from the Frontend Server exactly the JS file requested by + // build_runner. Frontend Server's recompilation logic will avoid + // extraneous recompilation. + await driver.recompileAndRecord( + sourceArg(webEntrypointAsset), + changedAssetUris, + [sourceArg(jsFESOutputId)], + ); + final outputFile = scratchSpace.fileFor(jsOutputId); + // Write an empty file if this output was deemed extraneous by FES. + if (!!await outputFile.exists()) { + await outputFile.create(recursive: true); + } + await scratchSpace.copyOutput(jsOutputId, buildStep); + await fixAndCopySourceMap( + ddcEntrypointId.changeExtension(jsSourceMapExtension), + scratchSpace, + buildStep, + ); + + // Copy the metadata output, modifying its contents to remove the temp + // directory from paths + final metadataId = ddcEntrypointId.changeExtension(metadataExtension); + final file = scratchSpace.fileFor(metadataId); + final content = await file.readAsString(); + final json = jsonDecode(content) as Map; + fixMetadataSources(json, scratchSpace.tempDir.uri); + await buildStep.writeAsString(metadataId, jsonEncode(json)); + } +} diff --git a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart index 65c73fe1a2..b8982e13dd 100644 --- a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart +++ b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:collection'; import 'dart:convert'; +import 'dart:io'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; @@ -12,8 +13,7 @@ import 'package:build_modules/build_modules.dart'; import 'package:path/path.dart' as _p; import 'package:pool/pool.dart'; -import 'ddc_names.dart'; -import 'dev_compiler_builder.dart'; +import 'common.dart'; import 'platforms.dart'; import 'web_entrypoint_builder.dart'; @@ -22,6 +22,10 @@ _p.Context get _context => _p.url; final _modulePartialExtension = _context.withoutExtension(jsModuleExtension); +final stackTraceMapperPath = + 'packages/build_web_compilers/src/' + 'dev_compiler_stack_trace/stack_trace_mapper.dart.js'; + /// Bootstraps a ddc application, creating the main entrypoint as well as the /// bootstrap and digest entrypoints. /// @@ -33,6 +37,7 @@ Future bootstrapDdc( Iterable requiredAssets = const [], String entrypointExtension = jsEntrypointExtension, required bool? nativeNullAssertions, + bool usesWebHotReload = false, }) async { platform = ddcPlatform; // Ensures that the sdk resources are built and available. @@ -47,7 +52,11 @@ Future bootstrapDdc( // First, ensure all transitive modules are built. List transitiveJsModules; try { - transitiveJsModules = await _ensureTransitiveJsModules(module, buildStep); + transitiveJsModules = await _ensureTransitiveJsModules( + module, + buildStep, + computeStronglyConnectedComponents: !usesWebHotReload, + ); } on UnsupportedModules catch (e) { final librariesString = (await e.exactLibraries(buildStep).toList()) .map( @@ -93,69 +102,124 @@ $librariesString _context.withoutExtension(buildStep.inputId.path), ); - // Map from module name to module path for custom modules. - final modulePaths = SplayTreeMap.of({ - 'dart_sdk': r'packages/build_web_compilers/src/dev_compiler/dart_sdk', - }); - for (final jsId in transitiveJsModules) { - // Strip out the top level dir from the path for any module, and set it to - // `packages/` for lib modules. We set baseUrl to `/` to simplify things, - // and we only allow you to serve top level directories. - final moduleName = ddcModuleName(jsId); - modulePaths[moduleName] = _context.withoutExtension( - jsId.path.startsWith('lib') - ? '$moduleName$jsModuleExtension' - : _context.joinAll(_context.split(jsId.path).skip(1)), - ); - } - final bootstrapId = dartEntrypointId.changeExtension(ddcBootstrapExtension); - final bootstrapModuleName = _context.withoutExtension( - _context.relative( - bootstrapId.path, - from: _context.dirname(dartEntrypointId.path), - ), + final bootstrapEndId = dartEntrypointId.changeExtension( + ddcBootstrapEndExtension, ); final dartEntrypointParts = _context.split(dartEntrypointId.path); - final entrypointLibraryName = _context.joinAll([ - // Convert to a package: uri for files under lib. - if (dartEntrypointParts.first == 'lib') - 'package:${module.primarySource.package}', - // Strip top-level directory from the path. - ...dartEntrypointParts.skip(1), - ]); - - final bootstrapContent = - StringBuffer('$_entrypointExtensionMarker\n(function() {\n') - ..write( - _dartLoaderSetup( - modulePaths, - _p.url.relative( - appDigestsOutput.path, - from: _p.url.dirname(bootstrapId.path), - ), - ), - ) - ..write(_requireJsConfig) - ..write( - _appBootstrap( - bootstrapModuleName: bootstrapModuleName, - entrypointLibraryName: entrypointLibraryName, - moduleName: appModuleName, - moduleScope: appModuleScope, - nativeNullAssertions: nativeNullAssertions, - oldModuleScope: oldAppModuleScope, - ), - ); + final packageName = module.primarySource.package; + final entrypointLibraryName = + usesWebHotReload + ? _context.joinAll([ + // Convert to a package: uri for files under lib. + if (dartEntrypointParts.first == 'lib') 'package:$packageName', + ...dartEntrypointParts, + ]) + : _context.joinAll([ + // Convert to a package: uri for files under lib. + if (dartEntrypointParts.first == 'lib') 'package:$packageName', + // Strip top-level directory from the path. + ...dartEntrypointParts.skip(1), + ]); + + final entrypointJsId = dartEntrypointId.changeExtension(entrypointExtension); - await buildStep.writeAsString(bootstrapId, bootstrapContent.toString()); + // Map from module name to module path for custom modules. + final modulePaths = SplayTreeMap(); + String entrypointJsContent; + String bootstrapContent; + String bootstrapEndContent; + if (usesWebHotReload) { + final ddcSdkUrl = + r'packages/build_web_compilers/src/dev_compiler/dart_sdk.js'; + modulePaths['dart_sdk'] = ddcSdkUrl; + for (final jsId in transitiveJsModules) { + // Strip out the top level dir from the path for any module, and set it to + // `packages/` for lib modules. We set baseUrl to `/` to simplify things, + // and we only allow you to serve top level directories. + final moduleName = ddcModuleName(jsId); + final libraryId = ddcLibraryId(jsId); + modulePaths[libraryId] = + jsId.path.startsWith('lib') + ? '$moduleName$jsModuleExtension' + : _context.joinAll(_context.split(jsId.path).skip(1)); + } + final bootstrapEndModuleName = _context.relative( + bootstrapId.path, + from: _context.dirname(bootstrapEndId.path), + ); + bootstrapContent = generateDDCLibraryBundleMainModule( + entrypoint: entrypointLibraryName, + nativeNullAssertions: nativeNullAssertions ?? false, + onLoadEndBootstrap: bootstrapEndModuleName, + ); + final bootstrapModuleName = _context.relative( + bootstrapId.path, + from: _context.dirname(dartEntrypointId.path), + ); + entrypointJsContent = generateDDCLibraryBundleBootstrapScript( + entrypoint: entrypointLibraryName, + ddcSdkUrl: ddcSdkUrl, + ddcModuleLoaderUrl: + 'packages/build_web_compilers/src/dev_compiler/ddc_module_loader.js', + mainBoostrapperUrl: bootstrapModuleName, + mapperUrl: stackTraceMapperPath, + isWindows: Platform.isWindows, + scriptIdsToPath: modulePaths, + ); + bootstrapEndContent = generateDDCLibraryBundleOnLoadEndBootstrap(); + } else { + modulePaths['dart_sdk'] = + r'packages/build_web_compilers/src/dev_compiler/dart_sdk'; + for (final jsId in transitiveJsModules) { + // Strip out the top level dir from the path for any module, and set it to + // `packages/` for lib modules. We set baseUrl to `/` to simplify things, + // and we only allow you to serve top level directories. + final moduleName = ddcModuleName(jsId); + modulePaths[moduleName] = _context.withoutExtension( + jsId.path.startsWith('lib') + ? '$moduleName$jsModuleExtension' + : _context.joinAll(_context.split(jsId.path).skip(1)), + ); + } + final bootstrapModuleName = _context.withoutExtension( + _context.relative( + bootstrapId.path, + from: _context.dirname(dartEntrypointId.path), + ), + ); + entrypointJsContent = _entryPointJs(bootstrapModuleName); + bootstrapContent = + (StringBuffer('$_entrypointExtensionMarker\n(function() {\n') + ..write( + _dartLoaderSetup( + modulePaths, + _p.url.relative( + appDigestsOutput.path, + from: _p.url.dirname(bootstrapId.path), + ), + ), + ) + ..write(_requireJsConfig) + ..write( + _appBootstrap( + bootstrapModuleName: bootstrapModuleName, + entrypointLibraryName: entrypointLibraryName, + moduleName: appModuleName, + moduleScope: appModuleScope, + nativeNullAssertions: nativeNullAssertions, + oldModuleScope: oldAppModuleScope, + ), + )) + .toString(); + // Unused for the AMD module system. + bootstrapEndContent = ''; + } - final entrypointJsContent = _entryPointJs(bootstrapModuleName); - await buildStep.writeAsString( - dartEntrypointId.changeExtension(entrypointExtension), - entrypointJsContent, - ); + await buildStep.writeAsString(entrypointJsId, entrypointJsContent); + await buildStep.writeAsString(bootstrapId, bootstrapContent); + await buildStep.writeAsString(bootstrapEndId, bootstrapEndContent); // Output the digests and merged_metadata for transitive modules. // These can be consumed for hot reloads and debugging. @@ -185,12 +249,14 @@ final _lazyBuildPool = Pool(16); /// unsupported modules. Future> _ensureTransitiveJsModules( Module module, - BuildStep buildStep, -) async { + BuildStep buildStep, { + bool computeStronglyConnectedComponents = true, +}) async { // Collect all the modules this module depends on, plus this module. final transitiveDeps = await module.computeTransitiveDependencies( buildStep, throwIfUnsupported: true, + computeStronglyConnectedComponents: computeStronglyConnectedComponents, ); final jsModules = [ @@ -277,8 +343,7 @@ String _entryPointJs(String bootstrapModuleName) => ''' $_currentDirectoryScript $_baseUrlScript - var mapperUri = baseUrl + "packages/build_web_compilers/src/" + - "dev_compiler_stack_trace/stack_trace_mapper.dart.js"; + var mapperUri = baseUrl + "$stackTraceMapperPath"; var requireUri = baseUrl + "packages/build_web_compilers/src/dev_compiler/require.js"; var mainUri = _currentDirectory + "$bootstrapModuleName"; @@ -570,3 +635,256 @@ Future _ensureResources( } } } + +const _simpleLoaderScript = r''' +window.$dartCreateScript = (function() { + // Find the nonce value. (Note, this is only computed once.) + var scripts = Array.from(document.getElementsByTagName("script")); + var nonce; + scripts.some( + script => (nonce = script.nonce || script.getAttribute("nonce"))); + // If present, return a closure that automatically appends the nonce. + if (nonce) { + return function() { + var script = document.createElement("script"); + script.nonce = nonce; + return script; + }; + } else { + return function() { + return document.createElement("script"); + }; + } +})(); + +// Loads a module [relativeUrl] relative to [root]. +// +// If not specified, [root] defaults to the directory serving the main app. +var forceLoadModule = function (relativeUrl, root) { + var actualRoot = root ?? _currentDirectory; + return new Promise(function(resolve, reject) { + var script = self.$dartCreateScript(); + let policy = { + createScriptURL: function(src) {return src;} + }; + if (self.trustedTypes && self.trustedTypes.createPolicy) { + policy = self.trustedTypes.createPolicy('dartDdcModuleUrl', policy); + } + script.onload = resolve; + script.onerror = reject; + script.src = policy.createScriptURL(actualRoot + relativeUrl); + document.head.appendChild(script); + }); +}; +'''; + +String generateDDCLibraryBundleBootstrapScript({ + required String entrypoint, + required String ddcSdkUrl, + required String ddcModuleLoaderUrl, + required String mainBoostrapperUrl, + required String mapperUrl, + required bool isWindows, + required Map scriptIdsToPath, +}) { + final scriptsJs = StringBuffer(); + scriptIdsToPath.forEach((id, path) { + scriptsJs.write('{"src": "$path", "id": "$id"},\n'); + }); + // Write the "true" main boostrapper last as part of the loader's convention. + scriptsJs.write('{"src": "$mainBoostrapperUrl", "id": "data-main"}\n'); + final boostrapScript = ''' +// Save the current directory so we can access it in a closure. +var _currentDirectory = (function () { + var _url = document.currentScript.src; + var lastSlash = _url.lastIndexOf('/'); + if (lastSlash == -1) return _url; + var currentDirectory = _url.substring(0, lastSlash + 1); + return currentDirectory; +})(); + +$_simpleLoaderScript + +(function() { + let appName = "$multiRootScheme:///$entrypoint"; + + // Load pre-requisite DDC scripts. We intentionally use invalid names to avoid + // namespace clashes. + let prerequisiteScripts = [ + { + "src": "$ddcModuleLoaderUrl", + "id": "ddc_module_loader \x00" + }, + { + "src": "$mapperUrl", + "id": "dart_stack_trace_mapper \x00" + } + ]; + + // Load ddc_module_loader.js to access DDC's module loader API. + let prerequisiteLoads = []; + for (let i = 0; i < prerequisiteScripts.length; i++) { + prerequisiteLoads.push(forceLoadModule(prerequisiteScripts[i].src)); + } + Promise.all(prerequisiteLoads).then((_) => afterPrerequisiteLogic()); + + // Save the current script so we can access it in a closure. + var _currentScript = document.currentScript; + + // Create a policy if needed to load the files during a hot restart. + let policy = { + createScriptURL: function(src) {return src;} + }; + if (self.trustedTypes && self.trustedTypes.createPolicy) { + policy = self.trustedTypes.createPolicy('dartDdcModuleUrl', policy); + } + + var afterPrerequisiteLogic = function() { + window.\$dartLoader.rootDirectories.push(_currentDirectory); + let scripts = [${scriptsJs.toString()}]; + + let loadConfig = new window.\$dartLoader.LoadConfiguration(); + // TODO(srujzs): Verify this is sufficient for Windows. + loadConfig.isWindows = $isWindows; + loadConfig.root = _currentDirectory; + loadConfig.bootstrapScript = scripts[scripts.length - 1]; + + loadConfig.loadScriptFn = function(loader) { + loader.addScriptsToQueue(scripts, null); + loader.loadEnqueuedModules(); + } + loadConfig.ddcEventForLoadStart = /* LOAD_ALL_MODULES_START */ 1; + loadConfig.ddcEventForLoadedOk = /* LOAD_ALL_MODULES_END_OK */ 2; + loadConfig.ddcEventForLoadedError = /* LOAD_ALL_MODULES_END_ERROR */ 3; + + let loader = new window.\$dartLoader.DDCLoader(loadConfig); + + // Record prerequisite scripts' fully resolved URLs. + prerequisiteScripts.forEach(script => loader.registerScript(script)); + + // Note: these variables should only be used in non-multi-app scenarios + // since they can be arbitrarily overridden based on multi-app load order. + window.\$dartLoader.loadConfig = loadConfig; + window.\$dartLoader.loader = loader; + + // Begin loading libraries + loader.nextAttempt(); + + // Set up stack trace mapper. + if (window.\$dartStackTraceUtility && + !window.\$dartStackTraceUtility.ready) { + window.\$dartStackTraceUtility.ready = true; + window.\$dartStackTraceUtility.setSourceMapProvider(function(url) { + var baseUrl = window.location.protocol + '//' + window.location.host; + url = url.replace(baseUrl + '/', ''); + if (url == 'dart_sdk.js') { + return dartDevEmbedder.debugger.getSourceMap('dart_sdk'); + } + url = url.replace(".lib.js", ""); + return dartDevEmbedder.debugger.getSourceMap(url); + }); + } + + let currentUri = _currentScript.src; + // We should have written a file containing all the scripts that need to be + // reloaded into the page. This is then read when a hot restart is triggered + // in DDC via the `\$dartReloadModifiedModules` callback. + let restartScripts = _currentDirectory + 'restart_scripts.json'; + + if (!window.\$dartReloadModifiedModules) { + window.\$dartReloadModifiedModules = (function(appName, callback) { + var xhttp = new XMLHttpRequest(); + xhttp.withCredentials = true; + xhttp.onreadystatechange = function() { + // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState + if (this.readyState == 4 && this.status == 200 || this.status == 304) { + var scripts = JSON.parse(this.responseText); + var numToLoad = 0; + var numLoaded = 0; + for (var i = 0; i < scripts.length; i++) { + var script = scripts[i]; + if (script.id == null) continue; + var src = script.src.toString(); + var oldSrc = window.\$dartLoader.moduleIdToUrl.get(script.id); + + // We might actually load from a different uri, delete the old one + // just to be sure. + window.\$dartLoader.urlToModuleId.delete(oldSrc); + + window.\$dartLoader.moduleIdToUrl.set(script.id, src); + window.\$dartLoader.urlToModuleId.set(src, script.id); + + numToLoad++; + + var el = document.getElementById(script.id); + if (el) el.remove(); + el = window.\$dartCreateScript(); + el.src = policy.createScriptURL(src); + el.async = false; + el.defer = true; + el.id = script.id; + el.onload = function() { + numLoaded++; + if (numToLoad == numLoaded) callback(); + }; + document.head.appendChild(el); + } + // Call `callback` right away if we found no updated scripts. + if (numToLoad == 0) callback(); + } + }; + xhttp.open("GET", restartScripts, true); + xhttp.send(); + }); + } + }; +})(); +'''; + return boostrapScript; +} + +String generateDDCLibraryBundleMainModule({ + required String entrypoint, + required bool nativeNullAssertions, + required String onLoadEndBootstrap, +}) { + // The typo below in "EXTENTION" is load-bearing, package:build depends on it. + return ''' +/* ENTRYPOINT_EXTENTION_MARKER */ + +(function() { + let appName = "$multiRootScheme:///$entrypoint"; + dartDevEmbedder.debugger.registerDevtoolsFormatter(); + + // Set up a final script that lets us know when all scripts have been loaded. + // Only then can we call the main method. + let onLoadEndSrc = '$onLoadEndBootstrap'; + window.\$dartLoader.loadConfig.bootstrapScript = { + src: onLoadEndSrc, + id: onLoadEndSrc, + }; + window.\$dartLoader.loadConfig.tryLoadBootstrapScript = true; + // Should be called by $onLoadEndBootstrap once all the scripts have been + // loaded. + window.$_onLoadEndCallback = function() { + let child = {}; + child.main = function() { + let sdkOptions = { + nativeNonNullAsserts: $nativeNullAssertions, + }; + dartDevEmbedder.runMain(appName, sdkOptions); + } + /* MAIN_EXTENSION_MARKER */ + child.main(); + } + // Call this immediately in build_web_compilers. + window.$_onLoadEndCallback(); +})(); +'''; +} + +String generateDDCLibraryBundleOnLoadEndBootstrap() { + return '''window.$_onLoadEndCallback();'''; +} + +const _onLoadEndCallback = r'$onLoadEndCallback'; diff --git a/build_web_compilers/lib/src/dev_compiler_builder.dart b/build_web_compilers/lib/src/dev_compiler_builder.dart index c4c5555f0f..66cbe235cd 100644 --- a/build_web_compilers/lib/src/dev_compiler_builder.dart +++ b/build_web_compilers/lib/src/dev_compiler_builder.dart @@ -11,19 +11,11 @@ import 'package:build/build.dart'; import 'package:build/experiments.dart'; import 'package:build_modules/build_modules.dart'; import 'package:path/path.dart' as p; -import 'package:scratch_space/scratch_space.dart'; import '../builders.dart'; import 'common.dart'; import 'errors.dart'; -final jsModuleErrorsExtension = '.ddc.js.errors'; -final jsModuleExtension = '.ddc.js'; -final jsSourceMapExtension = '.ddc.js.map'; -final metadataExtension = '.ddc.js.metadata'; -final symbolsExtension = '.ddc.js.symbols'; -final fullKernelExtension = '.ddc.full.dill'; - /// A builder which can output ddc modules! class DevCompilerBuilder implements Builder { final bool useIncrementalCompiler; @@ -50,6 +42,9 @@ class DevCompilerBuilder implements Builder { /// Enables canary features in DDC. final bool canaryFeatures; + /// Emits code with the DDC module system. + final bool ddcModules; + final bool trackUnusedInputs; final DartPlatform platform; @@ -79,6 +74,7 @@ class DevCompilerBuilder implements Builder { this.generateFullDill = false, this.emitDebugSymbols = false, this.canaryFeatures = false, + this.ddcModules = false, this.trackUnusedInputs = false, required this.platform, String? sdkKernelPath, @@ -133,6 +129,7 @@ class DevCompilerBuilder implements Builder { generateFullDill, emitDebugSymbols, canaryFeatures, + ddcModules, trackUnusedInputs, platformSdk, sdkKernelPath, @@ -155,6 +152,7 @@ Future _createDevCompilerModule( bool generateFullDill, bool emitDebugSymbols, bool canaryFeatures, + bool ddcModules, bool trackUnusedInputs, String dartSdk, String sdkKernelPath, @@ -204,7 +202,7 @@ Future _createDevCompilerModule( WorkRequest() ..arguments.addAll([ '--dart-sdk-summary=$sdkSummary', - '--modules=amd', + '--modules=${ddcModules ? 'ddc' : 'amd'}', '--no-summarize', if (generateFullDill) '--experimental-output-compiled-kernel', if (emitDebugSymbols) '--emit-debug-symbols', @@ -227,7 +225,7 @@ Future _createDevCompilerModule( ], if (usedInputsFile != null) '--used-inputs-file=${usedInputsFile.uri.toFilePath()}', - for (final source in module.sources) _sourceArg(source), + for (final source in module.sources) sourceArg(source), for (final define in environment.entries) '-D${define.key}=${define.value}', for (final experiment in enabledExperiments) @@ -303,7 +301,7 @@ Future _createDevCompilerModule( final file = scratchSpace.fileFor(metadataId); final content = await file.readAsString(); final json = jsonDecode(content) as Map; - _fixMetadataSources(json, scratchSpace.tempDir.uri); + fixMetadataSources(json, scratchSpace.tempDir.uri); await buildStep.writeAsString(metadataId, jsonEncode(json)); // Copy the symbols output, modifying its contents to remove the temp @@ -339,55 +337,3 @@ String _summaryArg(Module module) { ); return '--summary=${scratchSpace.fileFor(kernelAsset).path}=$moduleName'; } - -/// The url to compile for a source. -/// -/// Use the package: path for files under lib and the full absolute path for -/// other files. -String _sourceArg(AssetId id) { - final uri = canonicalUriFor(id); - return uri.startsWith('package:') ? uri : '$multiRootScheme:///${id.path}'; -} - -/// The module name according to ddc for [jsId] which represents the real js -/// module file. -String ddcModuleName(AssetId jsId) { - final jsPath = - jsId.path.startsWith('lib/') - ? jsId.path.replaceFirst('lib/', 'packages/${jsId.package}/') - : jsId.path; - return jsPath.substring(0, jsPath.length - jsModuleExtension.length); -} - -void _fixMetadataSources(Map json, Uri scratchUri) { - String updatePath(String path) => - Uri.parse(path).path.replaceAll(scratchUri.path, ''); - - final sourceMapUri = json['sourceMapUri'] as String?; - if (sourceMapUri != null) { - json['sourceMapUri'] = updatePath(sourceMapUri); - } - - final moduleUri = json['moduleUri'] as String?; - if (moduleUri != null) { - json['moduleUri'] = updatePath(moduleUri); - } - - final fullDillUri = json['fullDillUri'] as String?; - if (fullDillUri != null) { - json['fullDillUri'] = updatePath(fullDillUri); - } - - final libraries = json['libraries'] as List?; - if (libraries != null) { - for (final lib in libraries) { - final libraryJson = lib as Map?; - if (libraryJson != null) { - final fileUri = libraryJson['fileUri'] as String?; - if (fileUri != null) { - libraryJson['fileUri'] = updatePath(fileUri); - } - } - } - } -} diff --git a/build_web_compilers/lib/src/sdk_js_compile_builder.dart b/build_web_compilers/lib/src/sdk_js_compile_builder.dart index 77ae41a376..824fc89836 100644 --- a/build_web_compilers/lib/src/sdk_js_compile_builder.dart +++ b/build_web_compilers/lib/src/sdk_js_compile_builder.dart @@ -42,12 +42,17 @@ class SdkJsCompileBuilder implements Builder { /// Enables canary features in DDC. final bool canaryFeatures; + /// Emits DDC code with the Library Bundle module system, which supports hot + /// reload. + final bool usesWebHotReload; + SdkJsCompileBuilder({ required this.sdkKernelPath, required String outputPath, String? librariesPath, String? platformSdk, required this.canaryFeatures, + required this.usesWebHotReload, }) : platformSdk = platformSdk ?? sdkDir, librariesPath = librariesPath ?? @@ -72,6 +77,7 @@ class SdkJsCompileBuilder implements Builder { librariesPath, jsOutputId, canaryFeatures, + usesWebHotReload, ); } } @@ -84,6 +90,7 @@ Future _createDevCompilerModule( String librariesPath, AssetId jsOutputId, bool canaryFeatures, + bool usesWebHotReload, ) async { final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); final jsOutputFile = scratchSpace.fileFor(jsOutputId); @@ -112,8 +119,8 @@ Future _createDevCompilerModule( result = await Process.run(dartPath, [ snapshotPath, '--multi-root-scheme=org-dartlang-sdk', - '--modules=amd', - if (canaryFeatures) '--canary', + '--modules=${usesWebHotReload ? 'ddc' : 'amd'}', + if (canaryFeatures || usesWebHotReload) '--canary', '--module-name=dart_sdk', '-o', jsOutputFile.path, diff --git a/build_web_compilers/lib/src/sdk_js_copy_builder.dart b/build_web_compilers/lib/src/sdk_js_copy_builder.dart index e6432ad1b4..631cca43fb 100644 --- a/build_web_compilers/lib/src/sdk_js_copy_builder.dart +++ b/build_web_compilers/lib/src/sdk_js_copy_builder.dart @@ -10,12 +10,15 @@ import 'package:path/path.dart' as p; import 'common.dart'; -/// Copies the require.js file from the sdk itself, into the -/// build_web_compilers package at `lib/require.js`. +/// Copies the `require.js` and `ddc_module_loader.js` files from the SDK +/// into the `build_web_compilers` package under `lib/`. class SdkJsCopyBuilder implements Builder { @override final buildExtensions = { - r'$package$': ['lib/src/dev_compiler/require.js'], + r'$package$': [ + 'lib/src/dev_compiler/require.js', + 'lib/src/dev_compiler/ddc_module_loader.js', + ], }; /// Path to the require.js file that should be used for all ddc web apps. @@ -27,6 +30,16 @@ class SdkJsCopyBuilder implements Builder { 'require.js', ); + /// Path to the ddc_module_loader.js file that should be used for all ddc web + /// apps running with the library bundle module system. + final _sdkModuleLoaderJsLocation = p.join( + sdkDir, + 'lib', + 'dev_compiler', + 'ddc', + 'ddc_module_loader.js', + ); + @override FutureOr build(BuildStep buildStep) async { if (buildStep.inputId.package != 'build_web_compilers') { @@ -39,5 +52,12 @@ class SdkJsCopyBuilder implements Builder { AssetId('build_web_compilers', 'lib/src/dev_compiler/require.js'), await File(_sdkRequireJsLocation).readAsBytes(), ); + await buildStep.writeAsBytes( + AssetId( + 'build_web_compilers', + 'lib/src/dev_compiler/ddc_module_loader.js', + ), + await File(_sdkModuleLoaderJsLocation).readAsBytes(), + ); } } diff --git a/build_web_compilers/lib/src/web_entrypoint_builder.dart b/build_web_compilers/lib/src/web_entrypoint_builder.dart index d6d65a678b..11bbdd8c45 100644 --- a/build_web_compilers/lib/src/web_entrypoint_builder.dart +++ b/build_web_compilers/lib/src/web_entrypoint_builder.dart @@ -17,6 +17,7 @@ import 'dart2wasm_bootstrap.dart'; import 'dev_compiler_bootstrap.dart'; const ddcBootstrapExtension = '.dart.bootstrap.js'; +const ddcBootstrapEndExtension = '.dart.bootstrap.end.js'; const jsEntrypointExtension = '.dart.js'; const wasmExtension = '.wasm'; const wasmSourceMapExtension = '.wasm.map'; @@ -122,10 +123,16 @@ final class EntrypointBuilderOptions { /// necessary. final String? loaderExtension; + /// Whether or not to emit DDC entrypoints that support web hot reload. + /// + /// Web hot reload is only supported for DDC's Library Bundle module system. + final bool usesWebHotReload; + EntrypointBuilderOptions({ required this.compilers, this.nativeNullAssertions, this.loaderExtension, + this.usesWebHotReload = false, }); factory EntrypointBuilderOptions.fromOptions(BuilderOptions options) { @@ -137,6 +144,7 @@ final class EntrypointBuilderOptions { const dart2wasmArgsOption = 'dart2wasm_args'; const nativeNullAssertionsOption = 'native_null_assertions'; const loaderOption = 'loader'; + const webHotReloadOption = 'web-hot-reload'; String? defaultLoaderOption; const supportedOptions = [ @@ -146,11 +154,13 @@ final class EntrypointBuilderOptions { nativeNullAssertionsOption, dart2wasmArgsOption, loaderOption, + webHotReloadOption, ]; final config = options.config; final nativeNullAssertions = options.config[nativeNullAssertionsOption] as bool?; + final usesWebHotReload = options.config[webHotReloadOption] as bool?; final compilers = []; validateOptions( @@ -237,6 +247,7 @@ final class EntrypointBuilderOptions { config.containsKey(loaderOption) ? config[loaderOption] as String? : defaultLoaderOption, + usesWebHotReload: usesWebHotReload ?? false, ); } @@ -249,6 +260,7 @@ final class EntrypointBuilderOptions { '.dart': [ if (optionsFor(WebCompiler.DartDevc) case final ddc?) ...[ ddcBootstrapExtension, + ddcBootstrapEndExtension, mergedMetadataExtension, digestsEntrypointExtension, ddc.extension, @@ -315,10 +327,15 @@ class WebEntrypointBuilder implements Builder { compilationSteps.add( Future(() async { try { + final usesWebHotReload = options.usesWebHotReload; await bootstrapDdc( buildStep, nativeNullAssertions: options.nativeNullAssertions, - requiredAssets: _ddcSdkResources, + requiredAssets: + usesWebHotReload + ? _ddcLibraryBundleSdkResources + : _ddcSdkResources, + usesWebHotReload: usesWebHotReload, ); } on MissingModulesException catch (e) { log.severe('$e'); @@ -446,8 +463,16 @@ Future _isAppEntryPoint(AssetId dartId, AssetReader reader) async { } /// Files copied from the SDK that are required at runtime to run a DDC -/// application. +/// application with the AMD module system. final _ddcSdkResources = [ AssetId('build_web_compilers', 'lib/src/dev_compiler/dart_sdk.js'), AssetId('build_web_compilers', 'lib/src/dev_compiler/require.js'), ]; + +/// Files copied from the SDK that are required at runtime to run a DDC +/// application with the Library Bundle module system (which supports hot +/// reload). +final _ddcLibraryBundleSdkResources = [ + AssetId('build_web_compilers', 'lib/src/dev_compiler/dart_sdk.js'), + AssetId('build_web_compilers', 'lib/src/dev_compiler/ddc_module_loader.js'), +]; diff --git a/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart b/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart new file mode 100644 index 0000000000..c379a78875 --- /dev/null +++ b/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart @@ -0,0 +1,46 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; +import 'package:glob/glob.dart'; + +/// A builder that gathers information about a web target's 'main' entrypoint. +class WebEntrypointMarkerBuilder implements Builder { + @override + final buildExtensions = const { + r'$web$': ['.web.entrypoint.json'], + }; + + @override + Future build(BuildStep buildStep) async { + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + + final webAssets = await buildStep.findAssets(Glob('web/**')).toList(); + final webEntrypointJson = {}; + + for (final asset in webAssets) { + if (asset.extension == '.dart') { + final moduleLibrary = ModuleLibrary.fromSource( + asset, + await buildStep.readAsString(asset), + ); + if (moduleLibrary.hasMain && moduleLibrary.isEntryPoint) { + // We must save the main entrypoint as the recompilation target for + // the Frontend Server before any JS files are emitted. + scratchSpace.entrypointAssetId = asset; + webEntrypointJson['entrypoint'] = asset.uri.toString(); + break; + } + } + } + + await buildStep.writeAsString( + AssetId(buildStep.inputId.package, 'web/.web.entrypoint.json'), + jsonEncode(webEntrypointJson), + ); + } +} diff --git a/build_web_compilers/pubspec.yaml b/build_web_compilers/pubspec.yaml index 99ac045fd5..9b6e1a5d25 100644 --- a/build_web_compilers/pubspec.yaml +++ b/build_web_compilers/pubspec.yaml @@ -13,6 +13,7 @@ dependencies: bazel_worker: ^1.0.0 build: '>=2.0.0 <5.0.0' build_modules: ^5.0.0 + build_runner: ^2.0.0 collection: ^1.15.0 glob: ^2.0.0 logging: ^1.0.0 @@ -28,7 +29,6 @@ dev_dependencies: path: ../build_modules/test/fixtures/a b: path: ../build_modules/test/fixtures/b - build_runner: ^2.0.0 build_test: ^3.0.0 c: path: test/fixtures/c diff --git a/build_web_compilers/test/ddc_frontend_server_builder_test.dart b/build_web_compilers/test/ddc_frontend_server_builder_test.dart new file mode 100644 index 0000000000..1de9fbbf8d --- /dev/null +++ b/build_web_compilers/test/ddc_frontend_server_builder_test.dart @@ -0,0 +1,299 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@Tags(['integration']) +library; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; +import 'package:build_runner/src/build/asset_graph/graph.dart'; +import 'package:build_test/build_test.dart'; +import 'package:build_web_compilers/build_web_compilers.dart'; +import 'package:build_web_compilers/builders.dart'; +import 'package:build_web_compilers/src/ddc_frontend_server_builder.dart'; +import 'package:logging/logging.dart'; +import 'package:test/test.dart'; + +void main() { + group('DdcFrontendServerBuilder', () { + final assets = { + 'a|web/main.dart': ''' + import 'package:b/b.dart'; + void main() { + print(helloWorld); + } + ''', + 'b|lib/b.dart': "const helloWorld = 'Hello World!';", + }; + + final builders = [ + const ModuleLibraryBuilder(), + ddcMetaModuleBuilder(const BuilderOptions({'web-hot-reload': true})), + ddcModuleBuilder(const BuilderOptions({'web-hot-reload': true})), + DdcFrontendServerBuilder(), + ]; + + final allOutputs = { + 'a|web/main.module.library': isNotEmpty, + 'a|lib/.ddc.meta_module.raw': isNotEmpty, + 'a|web/main.ddc.module': isNotEmpty, + 'a|web/main.ddc.js': decodedMatches(contains('function main')), + 'a|web/main.ddc.js.map': isNotEmpty, + 'a|web/main.ddc.js.metadata': isNotEmpty, + 'b|lib/b.module.library': isNotEmpty, + 'b|lib/.ddc.meta_module.raw': isNotEmpty, + 'b|lib/b.ddc.module': isNotEmpty, + 'b|lib/b.ddc.js': decodedMatches(contains('Hello World!')), + 'b|lib/b.ddc.js.map': isNotEmpty, + 'b|lib/b.ddc.js.metadata': isNotEmpty, + }; + + // ignore: unused_local_variable + final nonJsOutputs = { + 'a|web/main.module.library': isNotEmpty, + 'a|lib/.ddc.meta_module.raw': isNotEmpty, + 'a|web/main.ddc.module': isNotEmpty, + 'b|lib/b.module.library': isNotEmpty, + 'b|lib/.ddc.meta_module.raw': isNotEmpty, + 'b|lib/b.ddc.module': isNotEmpty, + }; + + final aJsOutputs = { + 'a|web/main.ddc.js': decodedMatches(contains('function main')), + 'a|web/main.ddc.js.map': isNotEmpty, + 'a|web/main.ddc.js.metadata': isNotEmpty, + }; + + // ignore: unused_local_variable + final aModuleOutputs = { + 'a|web/main.module.library': isNotEmpty, + 'a|web/main.ddc.module': isNotEmpty, + }; + + final bJsOutputs = { + 'b|lib/b.ddc.js': decodedMatches(contains('Hello World!')), + 'b|lib/b.ddc.js.map': isNotEmpty, + 'b|lib/b.ddc.js.metadata': isNotEmpty, + }; + + final bModuleOutputs = { + 'b|lib/b.module.library': isNotEmpty, + 'b|lib/b.ddc.module': isNotEmpty, + }; + + setUp(() async { + final listener = Logger.root.onRecord.listen( + (r) => printOnFailure('$r\n${r.error}\n${r.stackTrace}'), + ); + addTearDown(listener.cancel); + }); + + /// [testBuilders] doesn't respect runs_before in build.yaml, so we must run + /// this before every invocation to ensure that side effects appear in the + /// right order. + Future runWebEntrypointMarker( + String rootPackage, { + TestReaderWriter? readerWriter, + AssetGraph? assetGraph, + + void Function(LogRecord log)? onLog, + }) async { + await testBuilder( + webEntrypointMarkerBuilder(const BuilderOptions({})), + assets, + outputs: {'$rootPackage|web/.web.entrypoint.json': isNotEmpty}, + rootPackage: rootPackage, + readerWriter: readerWriter, + assetGraph: assetGraph, + onLog: onLog, + performCleanup: false, + ); + } + + test('can compile a simple app', () async { + final logs = []; + await runWebEntrypointMarker('a', onLog: logs.add); + await testBuilders( + builders, + assets, + outputs: allOutputs, + // onLog: logs.add, + rootPackage: 'a', + performCleanup: true, + ); + + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + }); + + test('can recompile incrementally after valid edits', () async { + final logs = []; + await runWebEntrypointMarker('a'); + var result = await testBuilders( + builders, + assets, + outputs: allOutputs, + onLog: logs.add, + performCleanup: false, + rootPackage: 'a', + ); + + // Make a simple edit and re-run the build. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), ''' + const helloWorld = 'Hello Dash!'; + '''); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...aJsOutputs, + ...bJsOutputs, + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Dash!')), + 'b|lib/b.ddc.js': decodedMatches(contains('Hello Dash!')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: true, + ); + + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + }); + + test('can recompile incrementally after invalid edits', () async { + final logs = []; + await runWebEntrypointMarker('a'); + var result = await testBuilders( + builders, + assets, + outputs: { + ...allOutputs, + 'a|web/main.ddc.js': decodedMatches(contains('function main')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: false, + ); + + // Introduce a generic class and re-run the build. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), ''' + class Foo{} + const helloWorld = 'Hello Dash!'; + '''); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...aJsOutputs, + ...bJsOutputs, + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Dash!')), + 'b|lib/b.ddc.js': decodedMatches(contains('Hello Dash!')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: false, + ); + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + logs.clear(); + + // Change the number of generic parameters, which is invalid for hot + // reload. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), """ + class Foo{} + const helloWorld = 'Hello Python!'; + """); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Python!')), + 'a|web/main$jsModuleErrorsExtension': decodedMatches( + allOf( + contains('Exception'), + contains('Frontend Server encountered errors during compilation'), + contains('Limitation'), + ), + ), + 'b|lib/b$jsModuleErrorsExtension': decodedMatches( + allOf( + contains('Exception'), + contains('Frontend Server encountered errors during compilation'), + contains('Limitation'), + ), + ), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: false, + ); + + expect( + logs, + contains( + predicate( + (record) => + record.level == Level.SEVERE && + record.message.contains('Exception') && + record.message.contains( + 'Frontend Server encountered errors during compilation', + ) && + record.message.contains('Limitation'), + ), + ), + ); + logs.clear(); + + // Revert the number of generic parameters and successfully rebuild. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), """ + class Foo{} + const helloWorld = 'Hello Golang!'; + """); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...aJsOutputs, + ...bJsOutputs, + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Golang!')), + 'b|lib/b.ddc.js': decodedMatches(contains('Hello Golang!')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: true, + ); + + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + }); + }); +} diff --git a/scratch_space/lib/src/scratch_space.dart b/scratch_space/lib/src/scratch_space.dart index cf76e4bab6..6bcc69be68 100644 --- a/scratch_space/lib/src/scratch_space.dart +++ b/scratch_space/lib/src/scratch_space.dart @@ -21,12 +21,21 @@ class ScratchSpace { /// Whether or not this scratch space still exists. bool exists = true; + /// The built app's main entrypoint file. + /// + /// This must be set before any asset builders run when compiling with DDC and + /// hot reload. + late AssetId entrypointAssetId; + /// The `packages` directory under the temp directory. final Directory packagesDir; /// The temp directory at the root of this [ScratchSpace]. final Directory tempDir; + /// Holds all files that have been locally modified in this build. + final changedFilesInBuild = {}; + // Assets which have a file created but are still being written to. final _pendingWrites = >{}; @@ -92,6 +101,8 @@ class ScratchSpace { /// Copies [assetIds] to [tempDir] if they don't exist, using [reader] to /// read assets and mark dependencies. /// + /// Locally updated assets will be recorded in [changedFilesInBuild]. + /// /// Note that [BuildStep] implements [AssetReader] and that is typically /// what you will want to pass in. /// @@ -102,7 +113,6 @@ class ScratchSpace { if (!exists) { throw StateError('Tried to use a deleted ScratchSpace!'); } - final futures = assetIds.map((id) async { final digest = await reader.digest(id); @@ -111,6 +121,9 @@ class ScratchSpace { await _pendingWrites[id]; return; } + if (existing != null) { + changedFilesInBuild.add(id); + } _digests[id] = digest; try { @@ -140,6 +153,11 @@ class ScratchSpace { /// with [id] to make sure it is actually present. File fileFor(AssetId id) => File(p.join(tempDir.path, p.normalize(_relativePathFor(id)))); + + /// Performs cleanup required across builds. + void dispose() { + changedFilesInBuild.clear(); + } } /// Returns a canonical uri for [id].