From cd52372a2095e895248bae2664461122413d5057 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Thu, 28 Aug 2025 00:33:49 -0700 Subject: [PATCH 01/19] Initial support for DDC + FES with build_runner --- build_modules/lib/build_modules.dart | 12 +- build_modules/lib/src/common.dart | 11 + .../lib/src/ddc_names.dart | 36 +- .../lib/src/frontend_server_driver.dart | 516 ++++++++++++++++++ build_modules/lib/src/kernel_builder.dart | 47 +- build_modules/lib/src/module_builder.dart | 7 +- build_modules/lib/src/modules.dart | 18 +- build_modules/lib/src/scratch_space.dart | 1 + build_modules/lib/src/workers.dart | 53 +- build_modules/pubspec.yaml | 2 + build_web_compilers/build.yaml | 1 + build_web_compilers/lib/builders.dart | 31 +- build_web_compilers/lib/src/common.dart | 81 +++ .../lib/src/ddc_frontend_server_builder.dart | 104 ++++ .../lib/src/dev_compiler_bootstrap.dart | 446 ++++++++++++--- .../lib/src/dev_compiler_builder.dart | 185 +++---- .../lib/src/sdk_js_compile_builder.dart | 11 +- .../lib/src/sdk_js_copy_builder.dart | 26 +- .../lib/src/web_entrypoint_builder.dart | 29 +- 19 files changed, 1368 insertions(+), 249 deletions(-) rename {build_web_compilers => build_modules}/lib/src/ddc_names.dart (76%) create mode 100644 build_modules/lib/src/frontend_server_driver.dart create mode 100644 build_web_compilers/lib/src/ddc_frontend_server_builder.dart diff --git a/build_modules/lib/build_modules.dart b/build_modules/lib/build_modules.dart index 834333bb4d..332155c3aa 100644 --- a/build_modules/lib/build_modules.dart +++ b/build_modules/lib/build_modules.dart @@ -2,17 +2,23 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +export 'src/ddc_names.dart'; export 'src/errors.dart' show MissingModulesException, UnsupportedModules; -export 'src/kernel_builder.dart' - show KernelBuilder, multiRootScheme, reportUnusedKernelInputs; +export 'src/kernel_builder.dart' show KernelBuilder, reportUnusedKernelInputs; export 'src/meta_module_builder.dart' show MetaModuleBuilder, metaModuleExtension; export 'src/meta_module_clean_builder.dart' show MetaModuleCleanBuilder, metaModuleCleanExtension; export 'src/module_builder.dart' show ModuleBuilder, moduleExtension; +export 'src/module_library.dart' show ModuleLibrary; export 'src/module_library_builder.dart' show ModuleLibraryBuilder, moduleLibraryExtension; export 'src/modules.dart'; export 'src/platform.dart' show DartPlatform; export 'src/scratch_space.dart' show scratchSpace, scratchSpaceResource; -export 'src/workers.dart' show dartdevkDriverResource, maxWorkersPerTask; +export 'src/workers.dart' + show + dartdevkDriverResource, + frontendServerProxyDriverResource, + maxWorkersPerTask, + persistentFrontendServerResource; diff --git a/build_modules/lib/src/common.dart b/build_modules/lib/src/common.dart index 830ec63b08..03c8ebd4b6 100644 --- a/build_modules/lib/src/common.dart +++ b/build_modules/lib/src/common.dart @@ -2,9 +2,16 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + import 'package:build/build.dart'; +import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; +const multiRootScheme = 'org-dartlang-app'; +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); +final packagesFilePath = p.join('.dart_tool', 'package_config.json'); + final defaultAnalysisOptionsId = AssetId( 'build_modules', 'lib/src/analysis_options.default.yaml', @@ -16,6 +23,10 @@ String defaultAnalysisOptionsArg(ScratchSpace scratchSpace) => enum ModuleStrategy { fine, coarse } ModuleStrategy moduleStrategy(BuilderOptions options) { + // The DDC Library Bundle module system only supports fine modules. + if (options.config['web-hot-reload'] as bool? ?? false) { + return ModuleStrategy.fine; + } var config = options.config['strategy'] as String? ?? 'coarse'; switch (config) { case 'coarse': diff --git a/build_web_compilers/lib/src/ddc_names.dart b/build_modules/lib/src/ddc_names.dart similarity index 76% rename from build_web_compilers/lib/src/ddc_names.dart rename to build_modules/lib/src/ddc_names.dart index bdd60c60dc..b4afe29e3e 100644 --- a/build_web_compilers/lib/src/ddc_names.dart +++ b/build_modules/lib/src/ddc_names.dart @@ -1,14 +1,25 @@ -// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'package:path/path.dart' as p; +/// Logic in this file must be synchronized with their namesakes in DDC at: +/// pkg/dev_compiler/lib/src/compiler/js_names.dart + +bool isSdkInternalRuntimeUri(Uri importUri) { + return importUri.isScheme('dart') && importUri.path == '_runtime'; +} + +String libraryUriToJsIdentifier(Uri importUri) { + if (importUri.isScheme('dart')) { + return isSdkInternalRuntimeUri(importUri) ? 'dart' : importUri.path; + } + return pathToJSIdentifier(p.withoutExtension(importUri.pathSegments.last)); +} + /// Transforms a path to a valid JS identifier. /// -/// This logic must be synchronized with [pathToJSIdentifier] in DDC at: -/// pkg/dev_compiler/lib/src/compiler/module_builder.dart -/// /// For backwards compatibility, if this pattern is changed, /// dev_compiler_bootstrap.dart must be updated to accept both old and new /// patterns. @@ -35,12 +46,11 @@ String toJSIdentifier(String name) { for (var i = 0; i < name.length; i++) { var ch = name[i]; var needsEscape = ch == r'$' || _invalidCharInIdentifier.hasMatch(ch); - if (needsEscape && buffer == null) { - buffer = StringBuffer(name.substring(0, i)); - } - if (buffer != null) { - buffer.write(needsEscape ? '\$${ch.codeUnits.join("")}' : ch); + if (needsEscape) { + buffer ??= StringBuffer(name.substring(0, i)); } + + buffer?.write(needsEscape ? '\$${ch.codeUnits.join("")}' : ch); } var result = buffer != null ? '$buffer' : name; @@ -56,7 +66,11 @@ String toJSIdentifier(String name) { /// Also handles invalid variable names in strict mode, like "arguments". bool invalidVariableName(String keyword, {bool strictMode = true}) { switch (keyword) { - // http://www.ecma-international.org/ecma-262/6.0/#sec-future-reserved-words + // https: //262.ecma-international.org/6.0/#sec-reserved-words + case 'true': + case 'false': + case 'null': + // https://262.ecma-international.org/6.0/#sec-keywords case 'await': case 'break': case 'case': @@ -79,7 +93,6 @@ bool invalidVariableName(String keyword, {bool strictMode = true}) { case 'import': case 'in': case 'instanceof': - case 'let': case 'new': case 'return': case 'super': @@ -99,6 +112,7 @@ bool invalidVariableName(String keyword, {bool strictMode = true}) { // http://www.ecma-international.org/ecma-262/6.0/#sec-identifiers-static-semantics-early-errors case 'implements': case 'interface': + case 'let': case 'package': case 'private': case 'protected': diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart new file mode 100644 index 0000000000..77f4552fb5 --- /dev/null +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -0,0 +1,516 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:collection'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'common.dart'; +import 'ddc_names.dart' as ddc_names; + +final _log = Logger('FrontendServerProxy'); +final dartaotruntimePath = p.join(sdkDir, 'bin', 'dartaotruntime'); +final frontendServerSnapshotPath = p.join( + sdkDir, + 'bin', + 'snapshots', + 'frontend_server_aot.dart.snapshot', +); + +class FrontendServerProxyDriver { + PersistentFrontendServer? _frontendServer; + final _requestQueue = Queue<_CompilationRequest>(); + bool _isProcessing = false; + + void init(PersistentFrontendServer frontendServer) { + _frontendServer = frontendServer; + } + + Future requestModule(String entryPoint) async { + var compilerOutput = await compile(entryPoint); + if (compilerOutput == null) { + throw Exception('Frontend Server failed to compile $entryPoint'); + } + _frontendServer!.recordCompilerOutput(compilerOutput); + _frontendServer!.writeCompilerOutput(compilerOutput); + return compilerOutput; + } + + Future compile(String entryPoint) async { + final completer = Completer(); + _requestQueue.add(_CompileRequest(entryPoint, completer)); + if (!_isProcessing) _processQueue(); + return completer.future; + } + + Future recompile( + String entryPoint, + List invalidatedFiles, + ) async { + final completer = Completer(); + _requestQueue.add( + _RecompileRequest(entryPoint, invalidatedFiles, completer), + ); + if (!_isProcessing) _processQueue(); + return completer.future; + } + + void _processQueue() async { + if (_isProcessing || _requestQueue.isEmpty) return; + + _isProcessing = true; + final request = _requestQueue.removeFirst(); + CompilerOutput? output; + try { + if (request is _CompileRequest) { + output = await _frontendServer!.compile(request.entryPoint); + } else if (request is _RecompileRequest) { + output = await _frontendServer!.recompile( + request.entryPoint, + request.invalidatedFiles, + ); + } + if (output != null && output.errorCount == 0) { + _frontendServer!.accept(); + } + request.completer.complete(output); + } catch (e, s) { + request.completer.completeError(e, s); + } + + _isProcessing = false; + if (_requestQueue.isNotEmpty) { + _processQueue(); + } + } + + Future terminate() async { + await _frontendServer!.shutdown(); + _frontendServer = null; + } +} + +abstract class _CompilationRequest { + final Completer completer; + _CompilationRequest(this.completer); +} + +class _CompileRequest extends _CompilationRequest { + final String entryPoint; + _CompileRequest(this.entryPoint, super.completer); +} + +class _RecompileRequest extends _CompilationRequest { + final String entryPoint; + final List invalidatedFiles; + _RecompileRequest(this.entryPoint, this.invalidatedFiles, super.completer); +} + +class PersistentFrontendServer { + Process? _server; + final StdoutHandler _stdoutHandler; + final StreamController _stdinController; + final Uri outputDillUri; + final WebMemoryFilesystem _fileSystem; + CompilerOutput? _cachedOutput; + + PersistentFrontendServer._( + this._server, + this._stdoutHandler, + this._stdinController, + this.outputDillUri, + this._fileSystem, + ); + + static Future start({ + required String sdkRoot, + required Uri fileSystemRoot, + required Uri packagesFile, + }) async { + final outputDillUri = fileSystemRoot.resolve('output.dill'); + final platformDill = p.join(sdkDir, 'lib', '_internal', 'ddc_outline.dill'); + final args = [ + frontendServerSnapshotPath, + '--sdk-root=$sdkRoot', + '--incremental', + '--target=dartdevc', + '--dartdevc-module-format=ddc', + '--dartdevc-canary', + '--no-js-strongly-connected-components', + '--packages=${packagesFile.toFilePath()}', + '--experimental-emit-debug-metadata', + '--filesystem-scheme=$multiRootScheme', + '--filesystem-root=${fileSystemRoot.toFilePath()}', + '--platform=$platformDill', + '--output-dill=${outputDillUri.toFilePath()}', + '--output-incremental-dill=${outputDillUri.toFilePath()}', + ]; + final process = await Process.start(dartaotruntimePath, args); + var fileSystem = WebMemoryFilesystem(fileSystemRoot); + final stdoutHandler = StdoutHandler(logger: _log); + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen(stdoutHandler.handler); + process.stderr.transform(utf8.decoder).listen(stderr.writeln); + + final stdinController = StreamController(); + stdinController.stream.listen(process.stdin.writeln); + + return PersistentFrontendServer._( + process, + stdoutHandler, + stdinController, + outputDillUri, + fileSystem, + ); + } + + Future compile(String entryPoint) { + _stdoutHandler.reset(); + if (_cachedOutput != null) { + return Future.value(_cachedOutput); + } + _stdinController.add('compile $entryPoint'); + return _stdoutHandler.compilerOutput!.future.then( + (output) => _cachedOutput = output, + ); + } + + Future recompile( + String entryPoint, + List invalidatedFiles, + ) { + _stdoutHandler.reset(); + final inputKey = const Uuid().v4(); + _stdinController.add('recompile $entryPoint $inputKey'); + for (final file in invalidatedFiles) { + _stdinController.add(file.toString()); + } + _stdinController.add(inputKey); + return _stdoutHandler.compilerOutput!.future; + } + + void accept() { + _stdinController.add('accept'); + } + + void recordCompilerOutput(CompilerOutput output) { + if (output.errorCount != 0 || output.errorMessage != null) { + throw StateError('Attempting to record compiler output with errors.'); + } + final outputDillPath = outputDillUri.toFilePath(); + final codeFile = File('$outputDillPath.sources'); + final manifestFile = File('$outputDillPath.json'); + final sourcemapFile = File('$outputDillPath.map'); + final metadataFile = File('$outputDillPath.metadata'); + _fileSystem.update(codeFile, manifestFile, sourcemapFile, metadataFile); + } + + void writeCompilerOutput(CompilerOutput output) { + if (output.errorCount != 0 || output.errorMessage != null) { + throw StateError('Attempting to write compiler output with errors.'); + } + _fileSystem.writeToDisk(_fileSystem.jsRootUri); + } + + Future shutdown() async { + _stdinController.add('quit'); + await _stdinController.close(); + await _server?.exitCode; + _server = null; + } +} + +class CompilerOutput { + const CompilerOutput( + this.outputFilename, + this.errorCount, + this.sources, { + this.expressionData, + this.errorMessage, + }); + + final String outputFilename; + final int errorCount; + final List sources; + + /// Non-null for expression compilation requests. + final Uint8List? expressionData; + + /// Non-null when a compilation error was encountered. + final String? errorMessage; +} + +// A simple in-memory filesystem for handling frontend server's compiled +// output. +class WebMemoryFilesystem { + /// The root directory's URI from which JS file are being served. + final Uri jsRootUri; + + final Map files = {}; + final Map sourcemaps = {}; + final Map metadata = {}; + + // Holds all files that have already been written. + Set writtenFiles = {}; + + final List libraries = []; + + WebMemoryFilesystem(this.jsRootUri); + + /// Writes the entirety of this filesystem to [outputDirectoryUri]. + /// + /// [clearWritableState] Should only be set on a reload or restart. + void writeToDisk(Uri outputDirectoryUri, {bool clearWritableState = false}) { + assert( + Directory.fromUri(outputDirectoryUri).existsSync(), + '$outputDirectoryUri does not exist.', + ); + var filesToWrite = {...files, ...sourcemaps, ...metadata}; + filesToWrite.forEach((path, content) { + if (!writtenFiles.add(path)) { + // This file was already written between calls to `clearWritableState`. + return; + } + final outputFileUri = outputDirectoryUri.resolve(path); + var outputFilePath = outputFileUri.toFilePath().replaceFirst( + '.dart.lib.js', + '.ddc.js', + ); + final outputFile = File(outputFilePath); + outputFile.createSync(recursive: true); + outputFile.writeAsBytesSync(content); + }); + if (clearWritableState) { + files.clear(); + sourcemaps.clear(); + writtenFiles.clear(); + } + } + + /// Update the filesystem with the provided source and manifest files. + /// + /// Returns the list of updated files. + List update( + File codeFile, + File manifestFile, + File sourcemapFile, + File metadataFile, + ) { + final updatedFiles = []; + final codeBytes = codeFile.readAsBytesSync(); + final sourcemapBytes = sourcemapFile.readAsBytesSync(); + final manifest = Map.castFrom( + json.decode(manifestFile.readAsStringSync()) as Map, + ); + final metadataBytes = metadataFile.readAsBytesSync(); + + for (final filePath in manifest.keys) { + final fileUri = Uri.file(filePath); + final Map offsets = + Map.castFrom( + manifest[filePath] as Map, + ); + final codeOffsets = (offsets['code'] as List).cast(); + final sourcemapOffsets = + (offsets['sourcemap'] as List).cast(); + final metadataOffsets = + (offsets['metadata'] as List).cast(); + + if (codeOffsets.length != 2 || + sourcemapOffsets.length != 2 || + metadataOffsets.length != 2) { + _log.severe('Invalid manifest byte offsets: $offsets'); + continue; + } + + final codeStart = codeOffsets[0]; + final codeEnd = codeOffsets[1]; + if (codeStart < 0 || codeEnd > codeBytes.lengthInBytes) { + _log.severe('Invalid byte index: [$codeStart, $codeEnd]'); + continue; + } + + final byteView = Uint8List.view( + codeBytes.buffer, + codeStart, + codeEnd - codeStart, + ); + final fileName = + filePath.startsWith('/') ? filePath.substring(1) : filePath; + files[fileName] = byteView; + final moduleName = ddc_names.libraryUriToJsIdentifier(fileUri); + // TODO(markzipan): This is an overly simple heuristic to resolve the + // original Dart file. Replace this if it no longer holds. + var dartFileName = fileName; + if (dartFileName.endsWith('.lib.js')) { + dartFileName = fileName.substring( + 0, + fileName.length - '.lib.js'.length, + ); + } + final fullyResolvedFileUri = jsRootUri.resolve('$fileName'); + // TODO(markzipan): This is a simple hack to resolve kernel library URIs + // from JS files and might not generalize. + var libraryName = dartFileName; + if (libraryName.startsWith('packages/')) { + libraryName = + 'package:${libraryName.substring('packages/'.length, libraryName.length)}'; + } else { + libraryName = '$multiRootScheme:///$libraryName'; + } + final libraryInfo = LibraryInfo( + moduleName: moduleName, + libraryName: libraryName, + dartSourcePath: dartFileName, + jsSourcePath: fullyResolvedFileUri.toFilePath(), + ); + libraries.add(libraryInfo); + updatedFiles.add(fileName); + + final sourcemapStart = sourcemapOffsets[0]; + final sourcemapEnd = sourcemapOffsets[1]; + if (sourcemapStart < 0 || sourcemapEnd > sourcemapBytes.lengthInBytes) { + continue; + } + final sourcemapView = Uint8List.view( + sourcemapBytes.buffer, + sourcemapStart, + sourcemapEnd - sourcemapStart, + ); + final sourcemapName = '$fileName.map'; + sourcemaps[sourcemapName] = sourcemapView; + + final metadataStart = metadataOffsets[0]; + final metadataEnd = metadataOffsets[1]; + if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) { + _log.severe('Invalid byte index: [$metadataStart, $metadataEnd]'); + continue; + } + final metadataView = Uint8List.view( + metadataBytes.buffer, + metadataStart, + metadataEnd - metadataStart, + ); + metadata['$fileName.metadata'] = metadataView; + } + return updatedFiles; + } +} + +/// Bundles information associated with a DDC library. +class LibraryInfo { + final String moduleName; + final String libraryName; + final String dartSourcePath; + final String jsSourcePath; + + LibraryInfo({ + required this.moduleName, + required this.libraryName, + required this.dartSourcePath, + required this.jsSourcePath, + }); + + @override + String toString() => + 'LibraryInfo($moduleName, $libraryName, $dartSourcePath, $jsSourcePath)'; +} + +enum StdoutState { CollectDiagnostic, CollectDependencies } + +/// Handles stdin/stdout communication with the frontend server. +class StdoutHandler { + StdoutHandler({required Logger logger}) : _logger = logger { + reset(); + } + final Logger _logger; + + String? boundaryKey; + StdoutState state = StdoutState.CollectDiagnostic; + Completer? compilerOutput; + final sources = []; + + var _suppressCompilerMessages = false; + var _expectSources = true; + var _errorBuffer = StringBuffer(); + + void handler(String message) { + const kResultPrefix = 'result '; + if (boundaryKey == null && message.startsWith(kResultPrefix)) { + boundaryKey = message.substring(kResultPrefix.length); + return; + } + final messageBoundaryKey = boundaryKey; + if (messageBoundaryKey != null && message.startsWith(messageBoundaryKey)) { + if (_expectSources) { + if (state == StdoutState.CollectDiagnostic) { + state = StdoutState.CollectDependencies; + return; + } + } + if (message.length <= messageBoundaryKey.length) { + compilerOutput?.complete(); + return; + } + final spaceDelimiter = message.lastIndexOf(' '); + final fileName = message.substring( + messageBoundaryKey.length + 1, + spaceDelimiter, + ); + final errorCount = int.parse( + message.substring(spaceDelimiter + 1).trim(), + ); + + final output = CompilerOutput( + fileName, + errorCount, + sources, + expressionData: null, + errorMessage: _errorBuffer.isNotEmpty ? _errorBuffer.toString() : null, + ); + compilerOutput?.complete(output); + return; + } + switch (state) { + case StdoutState.CollectDiagnostic when _suppressCompilerMessages: + _logger.info(message); + _errorBuffer.writeln(message); + case StdoutState.CollectDiagnostic: + _logger.info(message); + _errorBuffer.writeln(message); + case StdoutState.CollectDependencies: + switch (message[0]) { + case '+': + sources.add(Uri.parse(message.substring(1))); + case '-': + sources.remove(Uri.parse(message.substring(1))); + default: + _logger.warning('Ignoring unexpected prefix for $message uri'); + } + } + } + + // This is needed to get ready to process next compilation result output, + // with its own boundary key and new completer. + void reset({ + bool suppressCompilerMessages = false, + bool expectSources = true, + bool readFile = false, + }) { + boundaryKey = null; + compilerOutput = Completer(); + _suppressCompilerMessages = suppressCompilerMessages; + _expectSources = expectSources; + state = StdoutState.CollectDiagnostic; + _errorBuffer = StringBuffer(); + } +} diff --git a/build_modules/lib/src/kernel_builder.dart b/build_modules/lib/src/kernel_builder.dart index 033e5add6d..9f9067011a 100644 --- a/build_modules/lib/src/kernel_builder.dart +++ b/build_modules/lib/src/kernel_builder.dart @@ -17,12 +17,11 @@ import 'package:scratch_space/scratch_space.dart'; import 'package:stream_transform/stream_transform.dart'; import '../build_modules.dart'; +import 'common.dart'; import 'errors.dart'; import 'module_cache.dart'; import 'workers.dart'; -const multiRootScheme = 'org-dartlang-app'; - /// A builder which can output kernel files for a given sdk. /// /// This creates kernel files based on [moduleExtension] files, which are what @@ -177,13 +176,12 @@ Future _createKernel({ await scratchSpace.ensureAssets(allAssetIds, buildStep); if (trackUnusedInputs) { - usedInputsFile = - await File( - p.join( - (await Directory.systemTemp.createTemp('kernel_builder_')).path, - 'used_inputs.txt', - ), - ).create(); + usedInputsFile = await File( + p.join( + (await Directory.systemTemp.createTemp('kernel_builder_')).path, + 'used_inputs.txt', + ), + ).create(); kernelInputPathToId = {}; } @@ -210,12 +208,11 @@ Future _createKernel({ var frontendWorker = await buildStep.fetchResource(frontendDriverResource); var response = await frontendWorker.doWork( request, - trackWork: - (response) => buildStep.trackStage( - 'Kernel Generate', - () => response, - isExternal: true, - ), + trackWork: (response) => buildStep.trackStage( + 'Kernel Generate', + () => response, + isExternal: true, + ), ); if (response.exitCode != EXIT_CODE_OK || !await outputFile.exists()) { throw KernelException( @@ -268,12 +265,11 @@ Future reportUnusedKernelInputs( if (usedPaths.isEmpty || usedPaths.first == '') return; String? firstMissingInputPath; - var usedIds = - usedPaths.map((usedPath) { - var id = inputPathToId[usedPath]; - if (id == null) firstMissingInputPath ??= usedPath; - return id; - }).toSet(); + var usedIds = usedPaths.map((usedPath) { + var id = inputPathToId[usedPath]; + if (id == null) firstMissingInputPath ??= usedPath; + return id; + }).toSet(); if (firstMissingInputPath != null) { log.warning( @@ -427,7 +423,7 @@ Future _addRequestArguments( request.arguments.addAll([ '--dart-sdk-summary=${Uri.file(p.join(sdkDir, sdkKernelPath))}', '--output=${outputFile.path}', - '--packages-file=$multiRootScheme:///${p.join('.dart_tool', 'package_config.json')}', + '--packages-file=$multiRootScheme:///$packagesFilePath', '--multi-root-scheme=$multiRootScheme', '--exclude-non-sources', summaryOnly ? '--summary-only' : '--no-summary-only', @@ -455,9 +451,8 @@ Future _addRequestArguments( } String _sourceArg(AssetId id) { - var uri = - id.path.startsWith('lib') - ? canonicalUriFor(id) - : '$multiRootScheme:///${id.path}'; + var uri = id.path.startsWith('lib') + ? canonicalUriFor(id) + : '$multiRootScheme:///${id.path}'; return '--source=$uri'; } diff --git a/build_modules/lib/src/module_builder.dart b/build_modules/lib/src/module_builder.dart index 85a621b5b6..f5f665e21c 100644 --- a/build_modules/lib/src/module_builder.dart +++ b/build_modules/lib/src/module_builder.dart @@ -7,6 +7,7 @@ import 'dart:async'; import 'package:build/build.dart'; import 'package:collection/collection.dart'; +import 'meta_module_builder.dart'; import 'meta_module_clean_builder.dart'; import 'module_cache.dart'; import 'module_library.dart'; @@ -32,12 +33,12 @@ class ModuleBuilder implements Builder { @override Future build(BuildStep buildStep) async { - final cleanMetaModules = await buildStep.fetchResource(metaModuleCache); + final metaModules = await buildStep.fetchResource(metaModuleCache); final metaModule = - (await cleanMetaModules.find( + (await metaModules.find( AssetId( buildStep.inputId.package, - 'lib/${metaModuleCleanExtension(_platform)}', + 'lib/${metaModuleExtension(_platform)}', ), buildStep, ))!; diff --git a/build_modules/lib/src/modules.dart b/build_modules/lib/src/modules.dart index 70962954c7..2747fdc53d 100644 --- a/build_modules/lib/src/modules.dart +++ b/build_modules/lib/src/modules.dart @@ -148,6 +148,7 @@ class Module { Future> computeTransitiveDependencies( BuildStep buildStep, { bool throwIfUnsupported = false, + bool computeStronglyConnectedComponents = true, }) async { final modules = await buildStep.fetchResource(moduleCache); var transitiveDeps = {}; @@ -183,13 +184,16 @@ class Module { if (throwIfUnsupported && unsupportedModules.isNotEmpty) { throw UnsupportedModules(unsupportedModules); } - var orderedModules = stronglyConnectedComponents( - transitiveDeps.values, - (m) => m.directDependencies.map((s) => transitiveDeps[s]!), - equals: (a, b) => a.primarySource == b.primarySource, - hashCode: (m) => m.primarySource.hashCode, - ); - return orderedModules.map((c) => c.single).toList(); + if (computeStronglyConnectedComponents) { + var orderedModules = stronglyConnectedComponents( + transitiveDeps.values, + (m) => m.directDependencies.map((s) => transitiveDeps[s]!), + equals: (a, b) => a.primarySource == b.primarySource, + hashCode: (m) => m.primarySource.hashCode, + ); + return orderedModules.map((c) => c.single).toList(); + } + return transitiveDeps.values.toList(); } } diff --git a/build_modules/lib/src/scratch_space.dart b/build_modules/lib/src/scratch_space.dart index ea7a5fd04e..c6bc8b423a 100644 --- a/build_modules/lib/src/scratch_space.dart +++ b/build_modules/lib/src/scratch_space.dart @@ -49,6 +49,7 @@ final scratchSpaceResource = Resource( // shut down before deleting it. await dartdevkWorkersAreDone; await frontendWorkersAreDone; + await frontendServerProxyWorkersAreDone; // Attempt to clean up the scratch space. Even after waiting for the workers // to shut down we might get file system exceptions on windows for an // arbitrary amount of time, so do retries with an exponential backoff. diff --git a/build_modules/lib/src/workers.dart b/build_modules/lib/src/workers.dart index c81b4989f1..15c83289ce 100644 --- a/build_modules/lib/src/workers.dart +++ b/build_modules/lib/src/workers.dart @@ -3,22 +3,23 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:math' show min; +import 'package:bazel_worker/bazel_worker.dart'; import 'package:bazel_worker/driver.dart'; import 'package:build/build.dart'; import 'package:path/path.dart' as p; +import 'common.dart'; +import 'frontend_server_driver.dart'; import 'scratch_space.dart'; -final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); - // If no terminal is attached, prevent a new one from launching. -final _processMode = - stdin.hasTerminal - ? ProcessStartMode.normal - : ProcessStartMode.detachedWithStdio; +final _processMode = stdin.hasTerminal + ? ProcessStartMode.normal + : ProcessStartMode.detachedWithStdio; /// Completes once the dartdevk workers have been shut down. Future get dartdevkWorkersAreDone => @@ -108,3 +109,43 @@ final frontendDriverResource = Resource( __frontendDriver = null; }, ); + +/// Completes once the Frontend Service proxy workers have been shut down. +Future get frontendServerProxyWorkersAreDone => + _frontendServerProxyWorkersAreDoneCompleter?.future ?? Future.value(); +Completer? _frontendServerProxyWorkersAreDoneCompleter; + +FrontendServerProxyDriver get _frontendServerProxyDriver { + _frontendServerProxyWorkersAreDoneCompleter ??= Completer(); + return __frontendServerProxyDriver ??= FrontendServerProxyDriver(); +} + +FrontendServerProxyDriver? __frontendServerProxyDriver; + +/// Manages a shared set of workers that proxy requests to a single +/// [persistentFrontendServerResource]. +final frontendServerProxyDriverResource = Resource( + () async => _frontendServerProxyDriver, + beforeExit: () async { + await __frontendServerProxyDriver?.terminate(); + _frontendServerProxyWorkersAreDoneCompleter?.complete(); + _frontendServerProxyWorkersAreDoneCompleter = null; + __frontendServerProxyDriver = null; + }, +); + +PersistentFrontendServer? __persistentFrontendServer; + +/// Manages a single persistent instance of the Frontend Server targeting DDC. +final persistentFrontendServerResource = Resource( + () async => + __persistentFrontendServer ??= await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: scratchSpace.tempDir.uri, + packagesFile: Uri.parse(packagesFilePath), + ), + beforeExit: () async { + await __persistentFrontendServer?.shutdown(); + __persistentFrontendServer = null; + }, +); diff --git a/build_modules/pubspec.yaml b/build_modules/pubspec.yaml index 5aeefc6acd..c811b5efc0 100644 --- a/build_modules/pubspec.yaml +++ b/build_modules/pubspec.yaml @@ -17,6 +17,7 @@ dependencies: build_config: ^1.0.0 collection: ^1.15.0 crypto: ^3.0.0 + file: ^7.0.1 glob: ^2.0.0 graphs: ^2.0.0 json_annotation: ^4.3.0 @@ -24,6 +25,7 @@ dependencies: path: ^1.8.0 scratch_space: ^1.0.0 stream_transform: ^2.0.0 + uuid: ^4.4.2 dev_dependencies: a: diff --git a/build_web_compilers/build.yaml b/build_web_compilers/build.yaml index 3f9c2eb887..e8c23af08b 100644 --- a/build_web_compilers/build.yaml +++ b/build_web_compilers/build.yaml @@ -24,6 +24,7 @@ builders: - lib/src/dev_compiler/dart_sdk.js - lib/src/dev_compiler/dart_sdk.js.map - lib/src/dev_compiler/require.js + - lib/src/dev_compiler/ddc_module_loader.js is_optional: True auto_apply: none runs_before: ["build_web_compilers:entrypoint"] diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index cf8e66f207..caddcd281a 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -8,6 +8,7 @@ import 'package:collection/collection.dart'; import 'build_web_compilers.dart'; import 'src/common.dart'; +import 'src/ddc_frontend_server_builder.dart'; import 'src/sdk_js_compile_builder.dart'; import 'src/sdk_js_copy_builder.dart'; @@ -15,7 +16,7 @@ import 'src/sdk_js_copy_builder.dart'; Builder webEntrypointBuilder(BuilderOptions options) => WebEntrypointBuilder.fromOptions(options); -// Ddc related builders +// DDC related builders Builder ddcMetaModuleBuilder(BuilderOptions options) => MetaModuleBuilder.forOptions(ddcPlatform, options); Builder ddcMetaModuleCleanBuilder(BuilderOptions _) => @@ -26,11 +27,23 @@ Builder ddcBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); _ensureSameDdcOptions(options); + if (_readWebHotReloadOption(options)) { + final entrypoint = _readEntrypoint(options); + if (entrypoint == null) { + throw StateError( + "DDC's Frontend Server configuration requires the " + '`entrypoint` to be specified in the `build.yaml`.', + ); + } + return DdcFrontendServerBuilder(entrypoint: entrypoint); + } + return DevCompilerBuilder( useIncrementalCompiler: _readUseIncrementalCompilerOption(options), generateFullDill: _readGenerateFullDillOption(options), emitDebugSymbols: _readEmitDebugSymbolsOption(options), canaryFeatures: _readCanaryOption(options), + ddcModules: _readWebHotReloadOption(options), sdkKernelPath: sdkDdcKernelPath, trackUnusedInputs: _readTrackInputsCompilerOption(options), platform: ddcPlatform, @@ -58,7 +71,9 @@ Builder sdkJsCopyRequirejs(BuilderOptions _) => SdkJsCopyBuilder(); Builder sdkJsCompile(BuilderOptions options) => SdkJsCompileBuilder( sdkKernelPath: 'lib/_internal/ddc_platform.dill', outputPath: 'lib/src/dev_compiler/dart_sdk.js', - canaryFeatures: _readCanaryOption(options), + canaryFeatures: + _readWebHotReloadOption(options) || _readCanaryOption(options), + usesWebHotReload: _readWebHotReloadOption(options), ); // Dart2js related builders @@ -135,6 +150,14 @@ bool _readTrackInputsCompilerOption(BuilderOptions options) { return options.config[_trackUnusedInputsCompilerOption] as bool? ?? true; } +String? _readEntrypoint(BuilderOptions options) { + return options.config[_entrypoint] as String?; +} + +bool _readWebHotReloadOption(BuilderOptions options) { + return options.config[_webHotReloadOption] as bool? ?? false; +} + Map _readEnvironmentOption(BuilderOptions options) { final environment = options.config[_environmentOption] as Map? ?? const {}; return environment.map((key, value) => MapEntry('$key', '$value')); @@ -147,6 +170,8 @@ const _emitDebugSymbolsOption = 'emit-debug-symbols'; const _canaryOption = 'canary'; const _trackUnusedInputsCompilerOption = 'track-unused-inputs'; const _environmentOption = 'environment'; +const _entrypoint = 'entrypoint'; +const _webHotReloadOption = 'web-hot-reload'; const _supportedOptions = [ _environmentOption, @@ -155,4 +180,6 @@ const _supportedOptions = [ _emitDebugSymbolsOption, _canaryOption, _trackUnusedInputsCompilerOption, + _entrypoint, + _webHotReloadOption, ]; diff --git a/build_web_compilers/lib/src/common.dart b/build_web_compilers/lib/src/common.dart index 13be1c300a..98198040cf 100644 --- a/build_web_compilers/lib/src/common.dart +++ b/build_web_compilers/lib/src/common.dart @@ -9,6 +9,14 @@ import 'package:build/build.dart'; import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; +final multiRootScheme = 'org-dartlang-app'; +final jsModuleErrorsExtension = '.ddc.js.errors'; +final jsModuleExtension = '.ddc.js'; +final jsSourceMapExtension = '.ddc.js.map'; +final metadataExtension = '.ddc.js.metadata'; +final symbolsExtension = '.ddc.js.symbols'; +final fullKernelExtension = '.ddc.full.dill'; + final defaultAnalysisOptionsId = AssetId( 'build_modules', 'lib/src/analysis_options.default.yaml', @@ -50,6 +58,15 @@ void validateOptions( } } +/// The url to compile for a source. +/// +/// Use the package: path for files under lib and the full absolute path for +/// other files. +String sourceArg(AssetId id) { + var uri = canonicalUriFor(id); + return uri.startsWith('package:') ? uri : '$multiRootScheme:///${id.path}'; +} + /// If [id] exists, assume it is a source map and fix up the source uris from /// it so they make sense in a browser context, then write the modified version /// using [writer]. @@ -85,3 +102,67 @@ Future fixAndCopySourceMap( await writer.writeAsString(id, jsonEncode(json)); } } + +void fixMetadataSources(Map json, Uri scratchUri) { + String updatePath(String path) => + Uri.parse(path).path.replaceAll(scratchUri.path, ''); + + var sourceMapUri = json['sourceMapUri'] as String?; + if (sourceMapUri != null) { + json['sourceMapUri'] = updatePath(sourceMapUri); + } + + var moduleUri = json['moduleUri'] as String?; + if (moduleUri != null) { + json['moduleUri'] = updatePath(moduleUri); + } + + var fullDillUri = json['fullDillUri'] as String?; + if (fullDillUri != null) { + json['fullDillUri'] = updatePath(fullDillUri); + } + + var libraries = json['libraries'] as List?; + if (libraries != null) { + for (var lib in libraries) { + var libraryJson = lib as Map?; + if (libraryJson != null) { + var fileUri = libraryJson['fileUri'] as String?; + if (fileUri != null) { + libraryJson['fileUri'] = updatePath(fileUri); + } + } + } + } +} + +/// The module name according to ddc for [jsId] which represents the real js +/// module file. +String ddcModuleName(AssetId jsId) { + var jsPath = + jsId.path.startsWith('lib/') + ? jsId.path.replaceFirst('lib/', 'packages/${jsId.package}/') + : jsId.path; + return jsPath.substring(0, jsPath.length - jsModuleExtension.length); +} + +String ddcLibraryId(AssetId jsId) { + var jsPath = + jsId.path.startsWith('lib/') + ? jsId.path.replaceFirst('lib/', 'package:${jsId.package}/') + : '$multiRootScheme://${jsId.path}'; + var prefix = jsPath.substring(0, jsPath.length - jsModuleExtension.length); + return '$prefix.dart'; +} + +AssetId changeAssetIdExtension( + AssetId inputId, + String inputExtension, + String outputExtension, +) { + assert(inputId.path.endsWith(inputExtension)); + var newPath = + inputId.path.substring(0, inputId.path.length - inputExtension.length) + + outputExtension; + return AssetId(inputId.package, newPath); +} diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart new file mode 100644 index 0000000000..a55daea866 --- /dev/null +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -0,0 +1,104 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; + +import 'common.dart'; +import 'errors.dart'; +import 'platforms.dart'; + +/// A builder which can compile DDC modules with the Frontend Server. +class DdcFrontendServerBuilder implements Builder { + final String entrypoint; + + DdcFrontendServerBuilder({required this.entrypoint}); + + @override + final Map> buildExtensions = { + moduleExtension(ddcPlatform): [ + jsModuleExtension, + jsModuleErrorsExtension, + jsSourceMapExtension, + metadataExtension, + ], + }; + + @override + Future build(BuildStep buildStep) async { + var moduleContents = await buildStep.readAsString(buildStep.inputId); + final module = Module.fromJson( + json.decode(moduleContents) as Map, + ); + var ddcEntrypointId = module.primarySource; + // Entrypoints always have a `.module` file for ease of looking them up, + // but they might not be the primary source. + if (ddcEntrypointId.changeExtension(moduleExtension(ddcPlatform)) != + buildStep.inputId) { + return; + } + + Future handleError(Object e) async { + await buildStep.writeAsString( + ddcEntrypointId.changeExtension(jsModuleErrorsExtension), + '$e', + ); + log.severe('Error encountered: $e'); + } + + try { + await _compile(module, buildStep); + } on DartDevcCompilationException catch (e) { + await handleError(e); + } on MissingModulesException catch (e) { + await handleError(e); + } + } + + /// Compile [module] with Frontend Server. + Future _compile(Module module, BuildStep buildStep) async { + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + await buildStep.trackStage( + 'EnsureAssets', + () => scratchSpace.ensureAssets(module.sources, buildStep), + ); + var ddcEntrypointId = module.primarySource; + var jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); + + final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); + + final frontendServer = await buildStep.fetchResource( + persistentFrontendServerResource, + ); + final driver = await buildStep.fetchResource( + frontendServerProxyDriverResource, + ); + driver.init(frontendServer); + await driver.requestModule(sourceArg(webEntrypointAsset)); + final outputFile = scratchSpace.fileFor(jsOutputId); + // Write an empty file if this file was deemed extraneous by the frontend + // server. + if (!!await outputFile.exists()) { + await outputFile.create(); + } + await scratchSpace.copyOutput(jsOutputId, buildStep); + await fixAndCopySourceMap( + ddcEntrypointId.changeExtension(jsSourceMapExtension), + scratchSpace, + buildStep, + ); + + // Copy the metadata output, modifying its contents to remove the temp + // directory from paths + var metadataId = ddcEntrypointId.changeExtension(metadataExtension); + var file = scratchSpace.fileFor(metadataId); + var content = await file.readAsString(); + var json = jsonDecode(content) as Map; + fixMetadataSources(json, scratchSpace.tempDir.uri); + await buildStep.writeAsString(metadataId, jsonEncode(json)); + } +} diff --git a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart index cc1e081683..656dd4fdfa 100644 --- a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart +++ b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart @@ -5,14 +5,16 @@ import 'dart:async'; import 'dart:collection'; import 'dart:convert'; +import 'dart:io'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; +import 'package:glob/glob.dart'; // ignore: no_leading_underscores_for_library_prefixes import 'package:path/path.dart' as _p; import 'package:pool/pool.dart'; -import 'ddc_names.dart'; +import 'common.dart'; import 'dev_compiler_builder.dart'; import 'platforms.dart'; import 'web_entrypoint_builder.dart'; @@ -22,6 +24,10 @@ _p.Context get _context => _p.url; final _modulePartialExtension = _context.withoutExtension(jsModuleExtension); +final stackTraceMapperPath = + "packages/build_web_compilers/src/" + "dev_compiler_stack_trace/stack_trace_mapper.dart.js"; + /// Bootstraps a ddc application, creating the main entrypoint as well as the /// bootstrap and digest entrypoints. /// @@ -33,6 +39,7 @@ Future bootstrapDdc( Iterable requiredAssets = const [], String entrypointExtension = jsEntrypointExtension, required bool? nativeNullAssertions, + bool usesWebHotReload = false, }) async { platform = ddcPlatform; // Ensures that the sdk resources are built and available. @@ -47,7 +54,11 @@ Future bootstrapDdc( // First, ensure all transitive modules are built. List transitiveJsModules; try { - transitiveJsModules = await _ensureTransitiveJsModules(module, buildStep); + transitiveJsModules = await _ensureTransitiveJsModules( + module, + buildStep, + computeStronglyConnectedComponents: !usesWebHotReload, + ); } on UnsupportedModules catch (e) { var librariesString = (await e.exactLibraries(buildStep).toList()) .map( @@ -95,69 +106,124 @@ https://github.com/dart-lang/build/blob/master/docs/faq.md#how-can-i-resolve-ski _context.withoutExtension(buildStep.inputId.path), ); - // Map from module name to module path for custom modules. - var modulePaths = SplayTreeMap.of({ - 'dart_sdk': r'packages/build_web_compilers/src/dev_compiler/dart_sdk', - }); - for (var jsId in transitiveJsModules) { - // Strip out the top level dir from the path for any module, and set it to - // `packages/` for lib modules. We set baseUrl to `/` to simplify things, - // and we only allow you to serve top level directories. - var moduleName = ddcModuleName(jsId); - modulePaths[moduleName] = _context.withoutExtension( - jsId.path.startsWith('lib') - ? '$moduleName$jsModuleExtension' - : _context.joinAll(_context.split(jsId.path).skip(1)), - ); - } - var bootstrapId = dartEntrypointId.changeExtension(ddcBootstrapExtension); - var bootstrapModuleName = _context.withoutExtension( - _context.relative( - bootstrapId.path, - from: _context.dirname(dartEntrypointId.path), - ), + var bootstrapEndId = dartEntrypointId.changeExtension( + ddcBootstrapEndExtension, ); var dartEntrypointParts = _context.split(dartEntrypointId.path); - var entrypointLibraryName = _context.joinAll([ - // Convert to a package: uri for files under lib. - if (dartEntrypointParts.first == 'lib') - 'package:${module.primarySource.package}', - // Strip top-level directory from the path. - ...dartEntrypointParts.skip(1), - ]); - - var bootstrapContent = - StringBuffer('$_entrypointExtensionMarker\n(function() {\n') - ..write( - _dartLoaderSetup( - modulePaths, - _p.url.relative( - appDigestsOutput.path, - from: _p.url.dirname(bootstrapId.path), - ), - ), - ) - ..write(_requireJsConfig) - ..write( - _appBootstrap( - bootstrapModuleName: bootstrapModuleName, - entrypointLibraryName: entrypointLibraryName, - moduleName: appModuleName, - moduleScope: appModuleScope, - nativeNullAssertions: nativeNullAssertions, - oldModuleScope: oldAppModuleScope, - ), - ); + var packageName = module.primarySource.package; + var entrypointLibraryName = + usesWebHotReload + ? _context.joinAll([ + // Convert to a package: uri for files under lib. + if (dartEntrypointParts.first == 'lib') 'package:$packageName', + ...dartEntrypointParts, + ]) + : _context.joinAll([ + // Convert to a package: uri for files under lib. + if (dartEntrypointParts.first == 'lib') 'package:$packageName', + // Strip top-level directory from the path. + ...dartEntrypointParts.skip(1), + ]); + + var entrypointJsId = dartEntrypointId.changeExtension(entrypointExtension); - await buildStep.writeAsString(bootstrapId, bootstrapContent.toString()); + // Map from module name to module path for custom modules. + var modulePaths = SplayTreeMap(); + String entrypointJsContent; + String bootstrapContent; + String bootstrapEndContent; + if (usesWebHotReload) { + final ddcSdkUrl = + r'packages/build_web_compilers/src/dev_compiler/dart_sdk.js'; + modulePaths['dart_sdk'] = ddcSdkUrl; + for (var jsId in transitiveJsModules) { + // Strip out the top level dir from the path for any module, and set it to + // `packages/` for lib modules. We set baseUrl to `/` to simplify things, + // and we only allow you to serve top level directories. + var moduleName = ddcModuleName(jsId); + var libraryId = ddcLibraryId(jsId); + modulePaths[libraryId] = + jsId.path.startsWith('lib') + ? '$moduleName$jsModuleExtension' + : _context.joinAll(_context.split(jsId.path).skip(1)); + } + var bootstrapEndModuleName = _context.relative( + bootstrapId.path, + from: _context.dirname(bootstrapEndId.path), + ); + bootstrapContent = generateDDCLibraryBundleMainModule( + entrypoint: entrypointLibraryName, + nativeNullAssertions: nativeNullAssertions ?? false, + onLoadEndBootstrap: bootstrapEndModuleName, + ); + var bootstrapModuleName = _context.relative( + bootstrapId.path, + from: _context.dirname(dartEntrypointId.path), + ); + entrypointJsContent = generateDDCLibraryBundleBootstrapScript( + entrypoint: entrypointLibraryName, + ddcSdkUrl: ddcSdkUrl, + ddcModuleLoaderUrl: + 'packages/build_web_compilers/src/dev_compiler/ddc_module_loader.js', + mainBoostrapperUrl: bootstrapModuleName, + mapperUrl: stackTraceMapperPath, + isWindows: Platform.isWindows, + scriptIdsToPath: modulePaths, + ); + bootstrapEndContent = generateDDCLibraryBundleOnLoadEndBootstrap(); + } else { + modulePaths['dart_sdk'] = + r'packages/build_web_compilers/src/dev_compiler/dart_sdk'; + for (var jsId in transitiveJsModules) { + // Strip out the top level dir from the path for any module, and set it to + // `packages/` for lib modules. We set baseUrl to `/` to simplify things, + // and we only allow you to serve top level directories. + var moduleName = ddcModuleName(jsId); + modulePaths[moduleName] = _context.withoutExtension( + jsId.path.startsWith('lib') + ? '$moduleName$jsModuleExtension' + : _context.joinAll(_context.split(jsId.path).skip(1)), + ); + } + var bootstrapModuleName = _context.withoutExtension( + _context.relative( + bootstrapId.path, + from: _context.dirname(dartEntrypointId.path), + ), + ); + entrypointJsContent = _entryPointJs(bootstrapModuleName); + bootstrapContent = + (StringBuffer('$_entrypointExtensionMarker\n(function() {\n') + ..write( + _dartLoaderSetup( + modulePaths, + _p.url.relative( + appDigestsOutput.path, + from: _p.url.dirname(bootstrapId.path), + ), + ), + ) + ..write(_requireJsConfig) + ..write( + _appBootstrap( + bootstrapModuleName: bootstrapModuleName, + entrypointLibraryName: entrypointLibraryName, + moduleName: appModuleName, + moduleScope: appModuleScope, + nativeNullAssertions: nativeNullAssertions, + oldModuleScope: oldAppModuleScope, + ), + )) + .toString(); + // Unused for the AMD module system. + bootstrapEndContent = ''; + } - var entrypointJsContent = _entryPointJs(bootstrapModuleName); - await buildStep.writeAsString( - dartEntrypointId.changeExtension(entrypointExtension), - entrypointJsContent, - ); + await buildStep.writeAsString(entrypointJsId, entrypointJsContent); + await buildStep.writeAsString(bootstrapId, bootstrapContent); + await buildStep.writeAsString(bootstrapEndId, bootstrapEndContent); // Output the digests and merged_metadata for transitive modules. // These can be consumed for hot reloads and debugging. @@ -187,12 +253,14 @@ final _lazyBuildPool = Pool(16); /// unsupported modules. Future> _ensureTransitiveJsModules( Module module, - BuildStep buildStep, -) async { + BuildStep buildStep, { + bool computeStronglyConnectedComponents = true, +}) async { // Collect all the modules this module depends on, plus this module. var transitiveDeps = await module.computeTransitiveDependencies( buildStep, throwIfUnsupported: true, + computeStronglyConnectedComponents: computeStronglyConnectedComponents, ); var jsModules = [ @@ -279,8 +347,7 @@ String _entryPointJs(String bootstrapModuleName) => ''' $_currentDirectoryScript $_baseUrlScript - var mapperUri = baseUrl + "packages/build_web_compilers/src/" + - "dev_compiler_stack_trace/stack_trace_mapper.dart.js"; + var mapperUri = baseUrl + "$stackTraceMapperPath"; var requireUri = baseUrl + "packages/build_web_compilers/src/dev_compiler/require.js"; var mainUri = _currentDirectory + "$bootstrapModuleName"; @@ -572,3 +639,256 @@ Future _ensureResources( } } } + +const _simpleLoaderScript = r''' +window.$dartCreateScript = (function() { + // Find the nonce value. (Note, this is only computed once.) + var scripts = Array.from(document.getElementsByTagName("script")); + var nonce; + scripts.some( + script => (nonce = script.nonce || script.getAttribute("nonce"))); + // If present, return a closure that automatically appends the nonce. + if (nonce) { + return function() { + var script = document.createElement("script"); + script.nonce = nonce; + return script; + }; + } else { + return function() { + return document.createElement("script"); + }; + } +})(); + +// Loads a module [relativeUrl] relative to [root]. +// +// If not specified, [root] defaults to the directory serving the main app. +var forceLoadModule = function (relativeUrl, root) { + var actualRoot = root ?? _currentDirectory; + return new Promise(function(resolve, reject) { + var script = self.$dartCreateScript(); + let policy = { + createScriptURL: function(src) {return src;} + }; + if (self.trustedTypes && self.trustedTypes.createPolicy) { + policy = self.trustedTypes.createPolicy('dartDdcModuleUrl', policy); + } + script.onload = resolve; + script.onerror = reject; + script.src = policy.createScriptURL(actualRoot + relativeUrl); + document.head.appendChild(script); + }); +}; +'''; + +String generateDDCLibraryBundleBootstrapScript({ + required String entrypoint, + required String ddcSdkUrl, + required String ddcModuleLoaderUrl, + required String mainBoostrapperUrl, + required String mapperUrl, + required bool isWindows, + required Map scriptIdsToPath, +}) { + var scriptsJs = StringBuffer(); + scriptIdsToPath.forEach((id, path) { + scriptsJs.write('{"src": "$path", "id": "$id"},\n'); + }); + // Write the "true" main boostrapper last as part of the loader's convention. + scriptsJs.write('{"src": "$mainBoostrapperUrl", "id": "data-main"}\n'); + var boostrapScript = ''' +// Save the current directory so we can access it in a closure. +var _currentDirectory = (function () { + var _url = document.currentScript.src; + var lastSlash = _url.lastIndexOf('/'); + if (lastSlash == -1) return _url; + var currentDirectory = _url.substring(0, lastSlash + 1); + return currentDirectory; +})(); + +$_simpleLoaderScript + +(function() { + let appName = "$multiRootScheme:///$entrypoint"; + + // Load pre-requisite DDC scripts. We intentionally use invalid names to avoid + // namespace clashes. + let prerequisiteScripts = [ + { + "src": "$ddcModuleLoaderUrl", + "id": "ddc_module_loader \x00" + }, + { + "src": "$mapperUrl", + "id": "dart_stack_trace_mapper \x00" + } + ]; + + // Load ddc_module_loader.js to access DDC's module loader API. + let prerequisiteLoads = []; + for (let i = 0; i < prerequisiteScripts.length; i++) { + prerequisiteLoads.push(forceLoadModule(prerequisiteScripts[i].src)); + } + Promise.all(prerequisiteLoads).then((_) => afterPrerequisiteLogic()); + + // Save the current script so we can access it in a closure. + var _currentScript = document.currentScript; + + // Create a policy if needed to load the files during a hot restart. + let policy = { + createScriptURL: function(src) {return src;} + }; + if (self.trustedTypes && self.trustedTypes.createPolicy) { + policy = self.trustedTypes.createPolicy('dartDdcModuleUrl', policy); + } + + var afterPrerequisiteLogic = function() { + window.\$dartLoader.rootDirectories.push(_currentDirectory); + let scripts = [${scriptsJs.toString()}]; + + let loadConfig = new window.\$dartLoader.LoadConfiguration(); + // TODO(srujzs): Verify this is sufficient for Windows. + loadConfig.isWindows = $isWindows; + loadConfig.root = _currentDirectory; + loadConfig.bootstrapScript = scripts[scripts.length - 1]; + + loadConfig.loadScriptFn = function(loader) { + loader.addScriptsToQueue(scripts, null); + loader.loadEnqueuedModules(); + } + loadConfig.ddcEventForLoadStart = /* LOAD_ALL_MODULES_START */ 1; + loadConfig.ddcEventForLoadedOk = /* LOAD_ALL_MODULES_END_OK */ 2; + loadConfig.ddcEventForLoadedError = /* LOAD_ALL_MODULES_END_ERROR */ 3; + + let loader = new window.\$dartLoader.DDCLoader(loadConfig); + + // Record prerequisite scripts' fully resolved URLs. + prerequisiteScripts.forEach(script => loader.registerScript(script)); + + // Note: these variables should only be used in non-multi-app scenarios + // since they can be arbitrarily overridden based on multi-app load order. + window.\$dartLoader.loadConfig = loadConfig; + window.\$dartLoader.loader = loader; + + // Begin loading libraries + loader.nextAttempt(); + + // Set up stack trace mapper. + if (window.\$dartStackTraceUtility && + !window.\$dartStackTraceUtility.ready) { + window.\$dartStackTraceUtility.ready = true; + window.\$dartStackTraceUtility.setSourceMapProvider(function(url) { + var baseUrl = window.location.protocol + '//' + window.location.host; + url = url.replace(baseUrl + '/', ''); + if (url == 'dart_sdk.js') { + return dartDevEmbedder.debugger.getSourceMap('dart_sdk'); + } + url = url.replace(".lib.js", ""); + return dartDevEmbedder.debugger.getSourceMap(url); + }); + } + + let currentUri = _currentScript.src; + // We should have written a file containing all the scripts that need to be + // reloaded into the page. This is then read when a hot restart is triggered + // in DDC via the `\$dartReloadModifiedModules` callback. + let restartScripts = _currentDirectory + 'restart_scripts.json'; + + if (!window.\$dartReloadModifiedModules) { + window.\$dartReloadModifiedModules = (function(appName, callback) { + var xhttp = new XMLHttpRequest(); + xhttp.withCredentials = true; + xhttp.onreadystatechange = function() { + // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState + if (this.readyState == 4 && this.status == 200 || this.status == 304) { + var scripts = JSON.parse(this.responseText); + var numToLoad = 0; + var numLoaded = 0; + for (var i = 0; i < scripts.length; i++) { + var script = scripts[i]; + if (script.id == null) continue; + var src = script.src.toString(); + var oldSrc = window.\$dartLoader.moduleIdToUrl.get(script.id); + + // We might actually load from a different uri, delete the old one + // just to be sure. + window.\$dartLoader.urlToModuleId.delete(oldSrc); + + window.\$dartLoader.moduleIdToUrl.set(script.id, src); + window.\$dartLoader.urlToModuleId.set(src, script.id); + + numToLoad++; + + var el = document.getElementById(script.id); + if (el) el.remove(); + el = window.\$dartCreateScript(); + el.src = policy.createScriptURL(src); + el.async = false; + el.defer = true; + el.id = script.id; + el.onload = function() { + numLoaded++; + if (numToLoad == numLoaded) callback(); + }; + document.head.appendChild(el); + } + // Call `callback` right away if we found no updated scripts. + if (numToLoad == 0) callback(); + } + }; + xhttp.open("GET", restartScripts, true); + xhttp.send(); + }); + } + }; +})(); +'''; + return boostrapScript; +} + +String generateDDCLibraryBundleMainModule({ + required String entrypoint, + required bool nativeNullAssertions, + required String onLoadEndBootstrap, +}) { + // The typo below in "EXTENTION" is load-bearing, package:build depends on it. + return ''' +/* ENTRYPOINT_EXTENTION_MARKER */ + +(function() { + let appName = "$multiRootScheme:///$entrypoint"; + dartDevEmbedder.debugger.registerDevtoolsFormatter(); + + // Set up a final script that lets us know when all scripts have been loaded. + // Only then can we call the main method. + let onLoadEndSrc = '$onLoadEndBootstrap'; + window.\$dartLoader.loadConfig.bootstrapScript = { + src: onLoadEndSrc, + id: onLoadEndSrc, + }; + window.\$dartLoader.loadConfig.tryLoadBootstrapScript = true; + // Should be called by $onLoadEndBootstrap once all the scripts have been + // loaded. + window.$_onLoadEndCallback = function() { + let child = {}; + child.main = function() { + let sdkOptions = { + nativeNonNullAsserts: $nativeNullAssertions, + }; + dartDevEmbedder.runMain(appName, sdkOptions); + } + /* MAIN_EXTENSION_MARKER */ + child.main(); + } + // Call this immediately in build_web_compilers. + window.$_onLoadEndCallback(); +})(); +'''; +} + +String generateDDCLibraryBundleOnLoadEndBootstrap() { + return '''window.$_onLoadEndCallback();'''; +} + +const _onLoadEndCallback = r'$onLoadEndCallback'; diff --git a/build_web_compilers/lib/src/dev_compiler_builder.dart b/build_web_compilers/lib/src/dev_compiler_builder.dart index 4b8d7a5cfb..3e768c9300 100644 --- a/build_web_compilers/lib/src/dev_compiler_builder.dart +++ b/build_web_compilers/lib/src/dev_compiler_builder.dart @@ -17,13 +17,6 @@ import '../builders.dart'; import 'common.dart'; import 'errors.dart'; -final jsModuleErrorsExtension = '.ddc.js.errors'; -final jsModuleExtension = '.ddc.js'; -final jsSourceMapExtension = '.ddc.js.map'; -final metadataExtension = '.ddc.js.metadata'; -final symbolsExtension = '.ddc.js.symbols'; -final fullKernelExtension = '.ddc.full.dill'; - /// A builder which can output ddc modules! class DevCompilerBuilder implements Builder { final bool useIncrementalCompiler; @@ -50,6 +43,9 @@ class DevCompilerBuilder implements Builder { /// Enables canary features in DDC. final bool canaryFeatures; + /// Emits code with the DDC module system. + final bool ddcModules; + final bool trackUnusedInputs; final DartPlatform platform; @@ -79,6 +75,7 @@ class DevCompilerBuilder implements Builder { this.generateFullDill = false, this.emitDebugSymbols = false, this.canaryFeatures = false, + this.ddcModules = false, this.trackUnusedInputs = false, required this.platform, String? sdkKernelPath, @@ -133,6 +130,7 @@ class DevCompilerBuilder implements Builder { generateFullDill, emitDebugSymbols, canaryFeatures, + ddcModules, trackUnusedInputs, platformSdk, sdkKernelPath, @@ -155,6 +153,7 @@ Future _createDevCompilerModule( bool generateFullDill, bool emitDebugSymbols, bool canaryFeatures, + bool ddcModules, bool trackUnusedInputs, String dartSdk, String sdkKernelPath, @@ -190,76 +189,72 @@ Future _createDevCompilerModule( File? usedInputsFile; if (trackUnusedInputs) { - usedInputsFile = - await File( - p.join( - (await Directory.systemTemp.createTemp('ddk_builder_')).path, - 'used_inputs.txt', - ), - ).create(); + usedInputsFile = await File( + p.join( + (await Directory.systemTemp.createTemp('ddk_builder_')).path, + 'used_inputs.txt', + ), + ).create(); kernelInputPathToId = {}; } - var request = - WorkRequest() - ..arguments.addAll([ - '--dart-sdk-summary=$sdkSummary', - '--modules=amd', - '--no-summarize', - if (generateFullDill) '--experimental-output-compiled-kernel', - if (emitDebugSymbols) '--emit-debug-symbols', - if (canaryFeatures) '--canary', - '-o', - jsOutputFile.path, - debugMode ? '--source-map' : '--no-source-map', - for (var dep in transitiveDeps) _summaryArg(dep), - '--packages=$multiRootScheme:///.dart_tool/package_config.json', - '--module-name=${ddcModuleName(jsId)}', - '--multi-root-scheme=$multiRootScheme', - '--multi-root=.', - '--track-widget-creation', - '--inline-source-map', - '--libraries-file=${p.toUri(librariesPath)}', - '--experimental-emit-debug-metadata', - if (useIncrementalCompiler) ...[ - '--reuse-compiler-result', - '--use-incremental-compiler', - ], - if (usedInputsFile != null) - '--used-inputs-file=${usedInputsFile.uri.toFilePath()}', - for (var source in module.sources) _sourceArg(source), - for (var define in environment.entries) - '-D${define.key}=${define.value}', - for (var experiment in enabledExperiments) - '--enable-experiment=$experiment', - ]) - ..inputs.add( - Input() - ..path = sdkSummary - ..digest = [0], - ) - ..inputs.addAll( - await Future.wait( - transitiveKernelDeps.map((dep) async { - var file = scratchSpace.fileFor(dep); - if (kernelInputPathToId != null) { - kernelInputPathToId[file.uri.toString()] = dep; - } - return Input() - ..path = file.path - ..digest = (await buildStep.digest(dep)).bytes; - }), - ), - ); + var request = WorkRequest() + ..arguments.addAll([ + '--dart-sdk-summary=$sdkSummary', + '--modules=${ddcModules ? 'ddc' : 'amd'}', + '--no-summarize', + if (generateFullDill) '--experimental-output-compiled-kernel', + if (emitDebugSymbols) '--emit-debug-symbols', + if (canaryFeatures) '--canary', + '-o', + jsOutputFile.path, + debugMode ? '--source-map' : '--no-source-map', + for (var dep in transitiveDeps) _summaryArg(dep), + '--packages=$multiRootScheme:///.dart_tool/package_config.json', + '--module-name=${ddcModuleName(jsId)}', + '--multi-root-scheme=$multiRootScheme', + '--multi-root=.', + '--track-widget-creation', + '--inline-source-map', + '--libraries-file=${p.toUri(librariesPath)}', + '--experimental-emit-debug-metadata', + if (useIncrementalCompiler) ...[ + '--reuse-compiler-result', + '--use-incremental-compiler', + ], + if (usedInputsFile != null) + '--used-inputs-file=${usedInputsFile.uri.toFilePath()}', + for (var source in module.sources) sourceArg(source), + for (var define in environment.entries) '-D${define.key}=${define.value}', + for (var experiment in enabledExperiments) + '--enable-experiment=$experiment', + ]) + ..inputs.add( + Input() + ..path = sdkSummary + ..digest = [0], + ) + ..inputs.addAll( + await Future.wait( + transitiveKernelDeps.map((dep) async { + var file = scratchSpace.fileFor(dep); + if (kernelInputPathToId != null) { + kernelInputPathToId[file.uri.toString()] = dep; + } + return Input() + ..path = file.path + ..digest = (await buildStep.digest(dep)).bytes; + }), + ), + ); try { var driverResource = dartdevkDriverResource; var driver = await buildStep.fetchResource(driverResource); var response = await driver.doWork( request, - trackWork: - (response) => - buildStep.trackStage('Compile', () => response, isExternal: true), + trackWork: (response) => + buildStep.trackStage('Compile', () => response, isExternal: true), ); // TODO(jakemac53): Fix the ddc worker mode so it always sends back a bad @@ -301,7 +296,7 @@ Future _createDevCompilerModule( var file = scratchSpace.fileFor(metadataId); var content = await file.readAsString(); var json = jsonDecode(content) as Map; - _fixMetadataSources(json, scratchSpace.tempDir.uri); + fixMetadataSources(json, scratchSpace.tempDir.uri); await buildStep.writeAsString(metadataId, jsonEncode(json)); // Copy the symbols output, modifying its contents to remove the temp @@ -335,55 +330,3 @@ String _summaryArg(Module module) { ); return '--summary=${scratchSpace.fileFor(kernelAsset).path}=$moduleName'; } - -/// The url to compile for a source. -/// -/// Use the package: path for files under lib and the full absolute path for -/// other files. -String _sourceArg(AssetId id) { - var uri = canonicalUriFor(id); - return uri.startsWith('package:') ? uri : '$multiRootScheme:///${id.path}'; -} - -/// The module name according to ddc for [jsId] which represents the real js -/// module file. -String ddcModuleName(AssetId jsId) { - var jsPath = - jsId.path.startsWith('lib/') - ? jsId.path.replaceFirst('lib/', 'packages/${jsId.package}/') - : jsId.path; - return jsPath.substring(0, jsPath.length - jsModuleExtension.length); -} - -void _fixMetadataSources(Map json, Uri scratchUri) { - String updatePath(String path) => - Uri.parse(path).path.replaceAll(scratchUri.path, ''); - - var sourceMapUri = json['sourceMapUri'] as String?; - if (sourceMapUri != null) { - json['sourceMapUri'] = updatePath(sourceMapUri); - } - - var moduleUri = json['moduleUri'] as String?; - if (moduleUri != null) { - json['moduleUri'] = updatePath(moduleUri); - } - - var fullDillUri = json['fullDillUri'] as String?; - if (fullDillUri != null) { - json['fullDillUri'] = updatePath(fullDillUri); - } - - var libraries = json['libraries'] as List?; - if (libraries != null) { - for (var lib in libraries) { - var libraryJson = lib as Map?; - if (libraryJson != null) { - var fileUri = libraryJson['fileUri'] as String?; - if (fileUri != null) { - libraryJson['fileUri'] = updatePath(fileUri); - } - } - } - } -} diff --git a/build_web_compilers/lib/src/sdk_js_compile_builder.dart b/build_web_compilers/lib/src/sdk_js_compile_builder.dart index e7c301edf1..dc18c519b7 100644 --- a/build_web_compilers/lib/src/sdk_js_compile_builder.dart +++ b/build_web_compilers/lib/src/sdk_js_compile_builder.dart @@ -42,12 +42,17 @@ class SdkJsCompileBuilder implements Builder { /// Enables canary features in DDC. final bool canaryFeatures; + /// Emits DDC code with the Library Bundle module system, which supports hot + /// reload. + final bool usesWebHotReload; + SdkJsCompileBuilder({ required this.sdkKernelPath, required String outputPath, String? librariesPath, String? platformSdk, required this.canaryFeatures, + required this.usesWebHotReload, }) : platformSdk = platformSdk ?? sdkDir, librariesPath = librariesPath ?? @@ -72,6 +77,7 @@ class SdkJsCompileBuilder implements Builder { librariesPath, jsOutputId, canaryFeatures, + usesWebHotReload, ); } } @@ -84,6 +90,7 @@ Future _createDevCompilerModule( String librariesPath, AssetId jsOutputId, bool canaryFeatures, + bool usesWebHotReload, ) async { var scratchSpace = await buildStep.fetchResource(scratchSpaceResource); var jsOutputFile = scratchSpace.fileFor(jsOutputId); @@ -112,8 +119,8 @@ Future _createDevCompilerModule( result = await Process.run(dartPath, [ snapshotPath, '--multi-root-scheme=org-dartlang-sdk', - '--modules=amd', - if (canaryFeatures) '--canary', + '--modules=${usesWebHotReload ? 'ddc' : 'amd'}', + if (canaryFeatures || usesWebHotReload) '--canary', '--module-name=dart_sdk', '-o', jsOutputFile.path, diff --git a/build_web_compilers/lib/src/sdk_js_copy_builder.dart b/build_web_compilers/lib/src/sdk_js_copy_builder.dart index e6432ad1b4..631cca43fb 100644 --- a/build_web_compilers/lib/src/sdk_js_copy_builder.dart +++ b/build_web_compilers/lib/src/sdk_js_copy_builder.dart @@ -10,12 +10,15 @@ import 'package:path/path.dart' as p; import 'common.dart'; -/// Copies the require.js file from the sdk itself, into the -/// build_web_compilers package at `lib/require.js`. +/// Copies the `require.js` and `ddc_module_loader.js` files from the SDK +/// into the `build_web_compilers` package under `lib/`. class SdkJsCopyBuilder implements Builder { @override final buildExtensions = { - r'$package$': ['lib/src/dev_compiler/require.js'], + r'$package$': [ + 'lib/src/dev_compiler/require.js', + 'lib/src/dev_compiler/ddc_module_loader.js', + ], }; /// Path to the require.js file that should be used for all ddc web apps. @@ -27,6 +30,16 @@ class SdkJsCopyBuilder implements Builder { 'require.js', ); + /// Path to the ddc_module_loader.js file that should be used for all ddc web + /// apps running with the library bundle module system. + final _sdkModuleLoaderJsLocation = p.join( + sdkDir, + 'lib', + 'dev_compiler', + 'ddc', + 'ddc_module_loader.js', + ); + @override FutureOr build(BuildStep buildStep) async { if (buildStep.inputId.package != 'build_web_compilers') { @@ -39,5 +52,12 @@ class SdkJsCopyBuilder implements Builder { AssetId('build_web_compilers', 'lib/src/dev_compiler/require.js'), await File(_sdkRequireJsLocation).readAsBytes(), ); + await buildStep.writeAsBytes( + AssetId( + 'build_web_compilers', + 'lib/src/dev_compiler/ddc_module_loader.js', + ), + await File(_sdkModuleLoaderJsLocation).readAsBytes(), + ); } } diff --git a/build_web_compilers/lib/src/web_entrypoint_builder.dart b/build_web_compilers/lib/src/web_entrypoint_builder.dart index 80ad6e1391..d99904095c 100644 --- a/build_web_compilers/lib/src/web_entrypoint_builder.dart +++ b/build_web_compilers/lib/src/web_entrypoint_builder.dart @@ -17,6 +17,7 @@ import 'dart2wasm_bootstrap.dart'; import 'dev_compiler_bootstrap.dart'; const ddcBootstrapExtension = '.dart.bootstrap.js'; +const ddcBootstrapEndExtension = '.dart.bootstrap.end.js'; const jsEntrypointExtension = '.dart.js'; const wasmExtension = '.wasm'; const wasmSourceMapExtension = '.wasm.map'; @@ -122,10 +123,16 @@ final class EntrypointBuilderOptions { /// necessary. final String? loaderExtension; + /// Whether or not to emit DDC entrypoints that support web hot reload. + /// + /// Web hot reload is only supported for DDC's Library Bundle module system. + final bool usesWebHotReload; + EntrypointBuilderOptions({ required this.compilers, this.nativeNullAssertions, this.loaderExtension, + this.usesWebHotReload = false, }); factory EntrypointBuilderOptions.fromOptions(BuilderOptions options) { @@ -137,6 +144,7 @@ final class EntrypointBuilderOptions { const dart2wasmArgsOption = 'dart2wasm_args'; const nativeNullAssertionsOption = 'native_null_assertions'; const loaderOption = 'loader'; + const webHotReloadOption = 'web-hot-reload'; String? defaultLoaderOption; const supportedOptions = [ @@ -146,11 +154,13 @@ final class EntrypointBuilderOptions { nativeNullAssertionsOption, dart2wasmArgsOption, loaderOption, + webHotReloadOption, ]; var config = options.config; var nativeNullAssertions = options.config[nativeNullAssertionsOption] as bool?; + var usesWebHotReload = options.config[webHotReloadOption] as bool?; var compilers = []; validateOptions( @@ -237,6 +247,7 @@ final class EntrypointBuilderOptions { config.containsKey(loaderOption) ? config[loaderOption] as String? : defaultLoaderOption, + usesWebHotReload: usesWebHotReload ?? false, ); } @@ -249,6 +260,7 @@ final class EntrypointBuilderOptions { '.dart': [ if (optionsFor(WebCompiler.DartDevc) case final ddc?) ...[ ddcBootstrapExtension, + ddcBootstrapEndExtension, mergedMetadataExtension, digestsEntrypointExtension, ddc.extension, @@ -315,10 +327,15 @@ class WebEntrypointBuilder implements Builder { compilationSteps.add( Future(() async { try { + var usesWebHotReload = options.usesWebHotReload; await bootstrapDdc( buildStep, nativeNullAssertions: options.nativeNullAssertions, - requiredAssets: _ddcSdkResources, + requiredAssets: + usesWebHotReload + ? _ddcLibraryBundleSdkResources + : _ddcSdkResources, + usesWebHotReload: usesWebHotReload, ); } on MissingModulesException catch (e) { log.severe('$e'); @@ -446,8 +463,16 @@ Future _isAppEntryPoint(AssetId dartId, AssetReader reader) async { } /// Files copied from the SDK that are required at runtime to run a DDC -/// application. +/// application with the AMD module system. final _ddcSdkResources = [ AssetId('build_web_compilers', 'lib/src/dev_compiler/dart_sdk.js'), AssetId('build_web_compilers', 'lib/src/dev_compiler/require.js'), ]; + +/// Files copied from the SDK that are required at runtime to run a DDC +/// application with the Library Bundle module system (which supports hot +/// reload). +final _ddcLibraryBundleSdkResources = [ + AssetId('build_web_compilers', 'lib/src/dev_compiler/dart_sdk.js'), + AssetId('build_web_compilers', 'lib/src/dev_compiler/ddc_module_loader.js'), +]; From af089bfac6f10c24ada0a0124fee1a80e8806d87 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 29 Aug 2025 00:26:13 -0700 Subject: [PATCH 02/19] Adding hot reload flag consistency checks and enforcing module builder fine modules for hot reload --- build_modules/lib/src/common.dart | 7 +- .../lib/src/frontend_server_driver.dart | 5 +- build_modules/lib/src/module_builder.dart | 17 +++-- build_modules/lib/src/workers.dart | 9 ++- build_web_compilers/lib/builders.dart | 65 +++++++++++++++---- .../lib/src/ddc_frontend_server_builder.dart | 14 ++-- .../lib/src/dev_compiler_bootstrap.dart | 6 +- 7 files changed, 86 insertions(+), 37 deletions(-) diff --git a/build_modules/lib/src/common.dart b/build_modules/lib/src/common.dart index 03c8ebd4b6..767fd93c4a 100644 --- a/build_modules/lib/src/common.dart +++ b/build_modules/lib/src/common.dart @@ -9,6 +9,7 @@ import 'package:path/path.dart' as p; import 'package:scratch_space/scratch_space.dart'; const multiRootScheme = 'org-dartlang-app'; +const webHotReloadOption = 'web-hot-reload'; final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); final packagesFilePath = p.join('.dart_tool', 'package_config.json'); @@ -23,8 +24,10 @@ String defaultAnalysisOptionsArg(ScratchSpace scratchSpace) => enum ModuleStrategy { fine, coarse } ModuleStrategy moduleStrategy(BuilderOptions options) { - // The DDC Library Bundle module system only supports fine modules. - if (options.config['web-hot-reload'] as bool? ?? false) { + // DDC's Library Bundle module system only supports fine modules since it must + // align with the Frontend Server's library management scheme. + var usesWebHotReload = options.config[webHotReloadOption] as bool? ?? false; + if (usesWebHotReload) { return ModuleStrategy.fine; } var config = options.config['strategy'] as String? ?? 'coarse'; diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index 77f4552fb5..33f3a61e1a 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -204,7 +204,10 @@ class PersistentFrontendServer { void recordCompilerOutput(CompilerOutput output) { if (output.errorCount != 0 || output.errorMessage != null) { - throw StateError('Attempting to record compiler output with errors.'); + throw StateError( + 'Attempting to record compiler output with errors: ' + '${output.errorMessage}', + ); } final outputDillPath = outputDillUri.toFilePath(); final codeFile = File('$outputDillPath.sources'); diff --git a/build_modules/lib/src/module_builder.dart b/build_modules/lib/src/module_builder.dart index f5f665e21c..5b9caa3447 100644 --- a/build_modules/lib/src/module_builder.dart +++ b/build_modules/lib/src/module_builder.dart @@ -23,7 +23,13 @@ String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; class ModuleBuilder implements Builder { final DartPlatform _platform; - ModuleBuilder(this._platform) + /// True if we use raw modules instead of clean modules. + /// + /// Clean meta modules are only used for DDC's AMD module system due its + /// requirement that self-referential libraries be bundled. + final bool useRawMetaModules; + + ModuleBuilder(this._platform, {this.useRawMetaModules = false}) : buildExtensions = { '.dart': [moduleExtension(_platform)], }; @@ -34,12 +40,13 @@ class ModuleBuilder implements Builder { @override Future build(BuildStep buildStep) async { final metaModules = await buildStep.fetchResource(metaModuleCache); + var metaModuleExtensionString = + useRawMetaModules + ? metaModuleExtension(_platform) + : metaModuleCleanExtension(_platform); final metaModule = (await metaModules.find( - AssetId( - buildStep.inputId.package, - 'lib/${metaModuleExtension(_platform)}', - ), + AssetId(buildStep.inputId.package, 'lib/$metaModuleExtensionString'), buildStep, ))!; var outputModule = metaModule.modules.firstWhereOrNull( diff --git a/build_modules/lib/src/workers.dart b/build_modules/lib/src/workers.dart index 15c83289ce..5cdefe6cd3 100644 --- a/build_modules/lib/src/workers.dart +++ b/build_modules/lib/src/workers.dart @@ -3,11 +3,9 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'dart:math' show min; -import 'package:bazel_worker/bazel_worker.dart'; import 'package:bazel_worker/driver.dart'; import 'package:build/build.dart'; import 'package:path/path.dart' as p; @@ -17,9 +15,10 @@ import 'frontend_server_driver.dart'; import 'scratch_space.dart'; // If no terminal is attached, prevent a new one from launching. -final _processMode = stdin.hasTerminal - ? ProcessStartMode.normal - : ProcessStartMode.detachedWithStdio; +final _processMode = + stdin.hasTerminal + ? ProcessStartMode.normal + : ProcessStartMode.detachedWithStdio; /// Completes once the dartdevk workers have been shut down. Future get dartdevkWorkersAreDone => diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index caddcd281a..8c255e248e 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -13,18 +13,33 @@ import 'src/sdk_js_compile_builder.dart'; import 'src/sdk_js_copy_builder.dart'; // Shared entrypoint builder -Builder webEntrypointBuilder(BuilderOptions options) => - WebEntrypointBuilder.fromOptions(options); +Builder webEntrypointBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return WebEntrypointBuilder.fromOptions(options); +} // DDC related builders -Builder ddcMetaModuleBuilder(BuilderOptions options) => - MetaModuleBuilder.forOptions(ddcPlatform, options); -Builder ddcMetaModuleCleanBuilder(BuilderOptions _) => - MetaModuleCleanBuilder(ddcPlatform); -Builder ddcModuleBuilder(BuilderOptions _) => ModuleBuilder(ddcPlatform); +Builder ddcMetaModuleBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return MetaModuleBuilder.forOptions(ddcPlatform, options); +} + +Builder ddcMetaModuleCleanBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return MetaModuleCleanBuilder(ddcPlatform); +} + +Builder ddcModuleBuilder(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return ModuleBuilder( + ddcPlatform, + useRawMetaModules: _readWebHotReloadOption(options), + ); +} Builder ddcBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); + _ensureSameDdcHotReloadOptions(options); _ensureSameDdcOptions(options); if (_readWebHotReloadOption(options)) { @@ -55,6 +70,7 @@ final ddcKernelExtension = '.ddc.dill'; Builder ddcKernelBuilder(BuilderOptions options) { validateOptions(options.config, _supportedOptions, 'build_web_compilers:ddc'); + _ensureSameDdcHotReloadOptions(options); _ensureSameDdcOptions(options); return KernelBuilder( @@ -68,13 +84,16 @@ Builder ddcKernelBuilder(BuilderOptions options) { } Builder sdkJsCopyRequirejs(BuilderOptions _) => SdkJsCopyBuilder(); -Builder sdkJsCompile(BuilderOptions options) => SdkJsCompileBuilder( - sdkKernelPath: 'lib/_internal/ddc_platform.dill', - outputPath: 'lib/src/dev_compiler/dart_sdk.js', - canaryFeatures: - _readWebHotReloadOption(options) || _readCanaryOption(options), - usesWebHotReload: _readWebHotReloadOption(options), -); +Builder sdkJsCompile(BuilderOptions options) { + _ensureSameDdcHotReloadOptions(options); + return SdkJsCompileBuilder( + sdkKernelPath: 'lib/_internal/ddc_platform.dill', + outputPath: 'lib/src/dev_compiler/dart_sdk.js', + canaryFeatures: + _readWebHotReloadOption(options) || _readCanaryOption(options), + usesWebHotReload: _readWebHotReloadOption(options), + ); +} // Dart2js related builders Builder dart2jsMetaModuleBuilder(BuilderOptions options) => @@ -130,6 +149,23 @@ void _ensureSameDdcOptions(BuilderOptions options) { } } +void _ensureSameDdcHotReloadOptions(BuilderOptions options) { + var webHotReload = _readWebHotReloadOption(options); + if (_lastWebHotReloadValue != null) { + if (webHotReload != _lastWebHotReloadValue) { + throw ArgumentError( + '`web-hot-reload` must be configured the same across the following ' + 'builders: build_web_compilers:ddc, build_web_compilers|sdk_js, ' + 'build_web_compilers|entrypoint, and build_web_compilers|ddc_modules.' + '\n\nPlease use the `global_options` section in ' + '`build.yaml` or the `--define` flag to set global options.', + ); + } + } else { + _lastWebHotReloadValue = webHotReload; + } +} + bool _readUseIncrementalCompilerOption(BuilderOptions options) { return options.config[_useIncrementalCompilerOption] as bool? ?? true; } @@ -164,6 +200,7 @@ Map _readEnvironmentOption(BuilderOptions options) { } Map? _previousDdcConfig; +bool? _lastWebHotReloadValue; const _useIncrementalCompilerOption = 'use-incremental-compiler'; const _generateFullDillOption = 'generate-full-dill'; const _emitDebugSymbolsOption = 'emit-debug-symbols'; diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index a55daea866..099b59c755 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -62,15 +62,18 @@ class DdcFrontendServerBuilder implements Builder { /// Compile [module] with Frontend Server. Future _compile(Module module, BuildStep buildStep) async { final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + // Resolve the 'real' entrypoint we'll pass to FES for compilation. + final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); await buildStep.trackStage( 'EnsureAssets', - () => scratchSpace.ensureAssets(module.sources, buildStep), + () => scratchSpace.ensureAssets([ + ...module.sources, + webEntrypointAsset, + ], buildStep), ); var ddcEntrypointId = module.primarySource; var jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); - final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); - final frontendServer = await buildStep.fetchResource( persistentFrontendServerResource, ); @@ -80,10 +83,9 @@ class DdcFrontendServerBuilder implements Builder { driver.init(frontendServer); await driver.requestModule(sourceArg(webEntrypointAsset)); final outputFile = scratchSpace.fileFor(jsOutputId); - // Write an empty file if this file was deemed extraneous by the frontend - // server. + // Write an empty file if this file was deemed extraneous by FES. if (!!await outputFile.exists()) { - await outputFile.create(); + await outputFile.create(recursive: true); } await scratchSpace.copyOutput(jsOutputId, buildStep); await fixAndCopySourceMap( diff --git a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart index 656dd4fdfa..61f02da98f 100644 --- a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart +++ b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart @@ -9,13 +9,11 @@ import 'dart:io'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; -import 'package:glob/glob.dart'; // ignore: no_leading_underscores_for_library_prefixes import 'package:path/path.dart' as _p; import 'package:pool/pool.dart'; import 'common.dart'; -import 'dev_compiler_builder.dart'; import 'platforms.dart'; import 'web_entrypoint_builder.dart'; @@ -25,8 +23,8 @@ _p.Context get _context => _p.url; final _modulePartialExtension = _context.withoutExtension(jsModuleExtension); final stackTraceMapperPath = - "packages/build_web_compilers/src/" - "dev_compiler_stack_trace/stack_trace_mapper.dart.js"; + 'packages/build_web_compilers/src/' + 'dev_compiler_stack_trace/stack_trace_mapper.dart.js'; /// Bootstraps a ddc application, creating the main entrypoint as well as the /// bootstrap and digest entrypoints. From 01e24118ea20629db90c8c65ef93a72d55d9c00f Mon Sep 17 00:00:00 2001 From: MarkZ Date: Thu, 4 Sep 2025 16:30:03 -0700 Subject: [PATCH 03/19] Moving sources to generated scratch dirs --- .../lib/src/frontend_server_driver.dart | 64 ++++++++++++------- build_modules/lib/src/modules.dart | 49 ++++++++++++++ build_modules/lib/src/scratch_space.dart | 8 +-- build_modules/lib/src/workers.dart | 2 +- .../lib/src/ddc_frontend_server_builder.dart | 12 +++- 5 files changed, 104 insertions(+), 31 deletions(-) diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index 33f3a61e1a..64837c8c5b 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// ignore_for_file: constant_identifier_names + import 'dart:async'; import 'dart:collection'; import 'dart:convert'; @@ -24,16 +26,19 @@ final frontendServerSnapshotPath = p.join( 'frontend_server_aot.dart.snapshot', ); +/// A driver that proxies build requests to a [PersistentFrontendServer] +/// instance. class FrontendServerProxyDriver { PersistentFrontendServer? _frontendServer; final _requestQueue = Queue<_CompilationRequest>(); bool _isProcessing = false; + CompilerOutput? _cachedOutput; void init(PersistentFrontendServer frontendServer) { _frontendServer = frontendServer; } - Future requestModule(String entryPoint) async { + Future compileAndRecord(String entryPoint) async { var compilerOutput = await compile(entryPoint); if (compilerOutput == null) { throw Exception('Frontend Server failed to compile $entryPoint'); @@ -43,6 +48,19 @@ class FrontendServerProxyDriver { return compilerOutput; } + Future recompileAndRecord( + String entryPoint, + List invalidatedFiles, + ) async { + var compilerOutput = await recompile(entryPoint, invalidatedFiles); + if (compilerOutput == null) { + throw Exception('Frontend Server failed to recompile $entryPoint'); + } + _frontendServer!.recordCompilerOutput(compilerOutput); + _frontendServer!.writeCompilerOutput(compilerOutput); + return compilerOutput; + } + Future compile(String entryPoint) async { final completer = Completer(); _requestQueue.add(_CompileRequest(entryPoint, completer)); @@ -70,12 +88,22 @@ class FrontendServerProxyDriver { CompilerOutput? output; try { if (request is _CompileRequest) { - output = await _frontendServer!.compile(request.entryPoint); + _cachedOutput = + output = await _frontendServer!.compile(request.entryPoint); } else if (request is _RecompileRequest) { - output = await _frontendServer!.recompile( - request.entryPoint, - request.invalidatedFiles, - ); + // Compile the first [_RecompileRequest] as a [_CompileRequest] to warm + // up the Frontend Server. + if (_cachedOutput == null) { + output = + _cachedOutput = await _frontendServer!.compile( + request.entryPoint, + ); + } else { + output = await _frontendServer!.recompile( + request.entryPoint, + request.invalidatedFiles, + ); + } } if (output != null && output.errorCount == 0) { _frontendServer!.accept(); @@ -113,13 +141,14 @@ class _RecompileRequest extends _CompilationRequest { _RecompileRequest(this.entryPoint, this.invalidatedFiles, super.completer); } +/// A single instance of the Frontend Server that persists across +/// compile/recompile requests. class PersistentFrontendServer { Process? _server; final StdoutHandler _stdoutHandler; final StreamController _stdinController; final Uri outputDillUri; final WebMemoryFilesystem _fileSystem; - CompilerOutput? _cachedOutput; PersistentFrontendServer._( this._server, @@ -175,13 +204,8 @@ class PersistentFrontendServer { Future compile(String entryPoint) { _stdoutHandler.reset(); - if (_cachedOutput != null) { - return Future.value(_cachedOutput); - } _stdinController.add('compile $entryPoint'); - return _stdoutHandler.compilerOutput!.future.then( - (output) => _cachedOutput = output, - ); + return _stdoutHandler.compilerOutput!.future; } Future recompile( @@ -252,7 +276,7 @@ class CompilerOutput { final String? errorMessage; } -// A simple in-memory filesystem for handling frontend server's compiled +// A simple in-memory filesystem for handling the Frontend Server's compiled // output. class WebMemoryFilesystem { /// The root directory's URI from which JS file are being served. @@ -262,9 +286,6 @@ class WebMemoryFilesystem { final Map sourcemaps = {}; final Map metadata = {}; - // Holds all files that have already been written. - Set writtenFiles = {}; - final List libraries = []; WebMemoryFilesystem(this.jsRootUri); @@ -279,10 +300,6 @@ class WebMemoryFilesystem { ); var filesToWrite = {...files, ...sourcemaps, ...metadata}; filesToWrite.forEach((path, content) { - if (!writtenFiles.add(path)) { - // This file was already written between calls to `clearWritableState`. - return; - } final outputFileUri = outputDirectoryUri.resolve(path); var outputFilePath = outputFileUri.toFilePath().replaceFirst( '.dart.lib.js', @@ -295,7 +312,6 @@ class WebMemoryFilesystem { if (clearWritableState) { files.clear(); sourcemaps.clear(); - writtenFiles.clear(); } } @@ -360,7 +376,7 @@ class WebMemoryFilesystem { fileName.length - '.lib.js'.length, ); } - final fullyResolvedFileUri = jsRootUri.resolve('$fileName'); + final fullyResolvedFileUri = jsRootUri.resolve(fileName); // TODO(markzipan): This is a simple hack to resolve kernel library URIs // from JS files and might not generalize. var libraryName = dartFileName; @@ -430,7 +446,7 @@ class LibraryInfo { enum StdoutState { CollectDiagnostic, CollectDependencies } -/// Handles stdin/stdout communication with the frontend server. +/// Handles stdin/stdout communication with the Frontend Server. class StdoutHandler { StdoutHandler({required Logger logger}) : _logger = logger { reset(); diff --git a/build_modules/lib/src/modules.dart b/build_modules/lib/src/modules.dart index 2747fdc53d..9f4f697917 100644 --- a/build_modules/lib/src/modules.dart +++ b/build_modules/lib/src/modules.dart @@ -195,6 +195,55 @@ class Module { } return transitiveDeps.values.toList(); } + /// Returns all [AssetId]s in the transitive dependencies of this module in + /// no specific order. + /// + /// Throws a [MissingModulesException] if there are any missing modules. This + /// typically means that somebody is trying to import a non-existing file. + /// + /// If [throwIfUnsupported] is `true`, then an [UnsupportedModules] + /// will be thrown if there are any modules that are not supported. + Future> computeTransitiveAssets( + BuildStep buildStep, { + bool throwIfUnsupported = false, + }) async { + final modules = await buildStep.fetchResource(moduleCache); + var transitiveDeps = {}; + var modulesToCrawl = {primarySource}; + var missingModuleSources = {}; + var unsupportedModules = {}; + var seenSources = {}; + + while (modulesToCrawl.isNotEmpty) { + var next = modulesToCrawl.last; + modulesToCrawl.remove(next); + if (transitiveDeps.containsKey(next)) continue; + var nextModuleId = next.changeExtension(moduleExtension(platform)); + var module = await modules.find(nextModuleId, buildStep); + if (module == null || module.isMissing) { + missingModuleSources.add(next); + continue; + } + if (throwIfUnsupported && !module.isSupported) { + unsupportedModules.add(module); + } + transitiveDeps[next] = module; + modulesToCrawl.addAll(module.directDependencies); + seenSources.addAll(module.sources); + } + + if (missingModuleSources.isNotEmpty) { + throw await MissingModulesException.create( + missingModuleSources, + transitiveDeps.values.toList()..add(this), + buildStep, + ); + } + if (throwIfUnsupported && unsupportedModules.isNotEmpty) { + throw UnsupportedModules(unsupportedModules); + } + return seenSources; + } } class _AssetIdConverter implements JsonConverter { diff --git a/build_modules/lib/src/scratch_space.dart b/build_modules/lib/src/scratch_space.dart index c6bc8b423a..a2dd576d6e 100644 --- a/build_modules/lib/src/scratch_space.dart +++ b/build_modules/lib/src/scratch_space.dart @@ -84,7 +84,7 @@ final scratchSpaceResource = Resource( }, ); -/// Modifies all package uris in [rootConfig] to work with the sctrach_space +/// Modifies all package uris in [rootConfig] to work with the scratch_space /// layout. These are uris of the form `../packages/`. /// /// Also modifies the `packageUri` for each package to be empty since the @@ -110,12 +110,12 @@ String _scratchSpacePackageConfig(String rootConfig, Uri packageConfigUri) { rootUri = rootUri.replace(path: '${rootUri.path}/'); } // We expect to see exactly one package where the root uri is equal to - // the current directory, and that is the current packge. + // the current directory, and that is the current package. if (rootUri == _currentDirUri) { assert(!foundRoot); foundRoot = true; - package['rootUri'] = '../'; - package['packageUri'] = '../packages/${package['name']}/'; + package['packageUri'] = ''; + package['rootUri'] = '../packages/${package['name']}/'; } else { package['rootUri'] = '../packages/${package['name']}/'; package.remove('packageUri'); diff --git a/build_modules/lib/src/workers.dart b/build_modules/lib/src/workers.dart index 5cdefe6cd3..bc11811704 100644 --- a/build_modules/lib/src/workers.dart +++ b/build_modules/lib/src/workers.dart @@ -141,7 +141,7 @@ final persistentFrontendServerResource = Resource( __persistentFrontendServer ??= await PersistentFrontendServer.start( sdkRoot: sdkDir, fileSystemRoot: scratchSpace.tempDir.uri, - packagesFile: Uri.parse(packagesFilePath), + packagesFile: scratchSpace.tempDir.uri.resolve(packagesFilePath), ), beforeExit: () async { await __persistentFrontendServer?.shutdown(); diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 099b59c755..8b98dd3703 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -61,13 +61,17 @@ class DdcFrontendServerBuilder implements Builder { /// Compile [module] with Frontend Server. Future _compile(Module module, BuildStep buildStep) async { + var transitiveAssets = await buildStep.trackStage( + 'CollectTransitiveDeps', + () => module.computeTransitiveAssets(buildStep), + ); final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); // Resolve the 'real' entrypoint we'll pass to FES for compilation. final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); await buildStep.trackStage( 'EnsureAssets', () => scratchSpace.ensureAssets([ - ...module.sources, + ...transitiveAssets, webEntrypointAsset, ], buildStep), ); @@ -81,7 +85,11 @@ class DdcFrontendServerBuilder implements Builder { frontendServerProxyDriverResource, ); driver.init(frontendServer); - await driver.requestModule(sourceArg(webEntrypointAsset)); + final invalidatedFileUris = [for (var source in module.sources) source.uri]; + await driver.recompileAndRecord( + sourceArg(ddcEntrypointId), + invalidatedFileUris, + ); final outputFile = scratchSpace.fileFor(jsOutputId); // Write an empty file if this file was deemed extraneous by FES. if (!!await outputFile.exists()) { From fa6594f79be377c76303d0cbaea46f408609ba01 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 5 Sep 2025 14:46:31 -0700 Subject: [PATCH 04/19] Only recompiling and invalidating requested source files --- .../lib/src/frontend_server_driver.dart | 132 ++++++++++++------ .../lib/src/ddc_frontend_server_builder.dart | 19 +-- 2 files changed, 97 insertions(+), 54 deletions(-) diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index 64837c8c5b..812750a218 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -38,43 +38,49 @@ class FrontendServerProxyDriver { _frontendServer = frontendServer; } - Future compileAndRecord(String entryPoint) async { - var compilerOutput = await compile(entryPoint); - if (compilerOutput == null) { - throw Exception('Frontend Server failed to compile $entryPoint'); - } - _frontendServer!.recordCompilerOutput(compilerOutput); - _frontendServer!.writeCompilerOutput(compilerOutput); - return compilerOutput; - } - + /// Sends a recompile request to the Frontend Server at [entrypoint] with + /// [invalidatedFiles]. + /// + /// The initial recompile request is treated as a full compile. + /// + /// [filesToWrite] contains JS files that should be written to the filesystem. + /// If left empty, all files are written. Future recompileAndRecord( - String entryPoint, + String entrypoint, List invalidatedFiles, + Iterable filesToWrite, ) async { - var compilerOutput = await recompile(entryPoint, invalidatedFiles); + var compilerOutput = await recompile(entrypoint, invalidatedFiles); if (compilerOutput == null) { - throw Exception('Frontend Server failed to recompile $entryPoint'); + throw Exception('Frontend Server failed to recompile $entrypoint'); + } + if (compilerOutput.errorCount != 0 || compilerOutput.errorMessage != null) { + throw Exception( + 'Frontend Server encountered errors during compilation: ' + '${compilerOutput.errorMessage}', + ); + } + _frontendServer!.recordFiles(); + for (var file in filesToWrite) { + _frontendServer!.writeFile(file); } - _frontendServer!.recordCompilerOutput(compilerOutput); - _frontendServer!.writeCompilerOutput(compilerOutput); return compilerOutput; } - Future compile(String entryPoint) async { + Future compile(String entrypoint) async { final completer = Completer(); - _requestQueue.add(_CompileRequest(entryPoint, completer)); + _requestQueue.add(_CompileRequest(entrypoint, completer)); if (!_isProcessing) _processQueue(); return completer.future; } Future recompile( - String entryPoint, + String entrypoint, List invalidatedFiles, ) async { final completer = Completer(); _requestQueue.add( - _RecompileRequest(entryPoint, invalidatedFiles, completer), + _RecompileRequest(entrypoint, invalidatedFiles, completer), ); if (!_isProcessing) _processQueue(); return completer.future; @@ -89,18 +95,18 @@ class FrontendServerProxyDriver { try { if (request is _CompileRequest) { _cachedOutput = - output = await _frontendServer!.compile(request.entryPoint); + output = await _frontendServer!.compile(request.entrypoint); } else if (request is _RecompileRequest) { // Compile the first [_RecompileRequest] as a [_CompileRequest] to warm // up the Frontend Server. if (_cachedOutput == null) { output = _cachedOutput = await _frontendServer!.compile( - request.entryPoint, + request.entrypoint, ); } else { output = await _frontendServer!.recompile( - request.entryPoint, + request.entrypoint, request.invalidatedFiles, ); } @@ -131,14 +137,14 @@ abstract class _CompilationRequest { } class _CompileRequest extends _CompilationRequest { - final String entryPoint; - _CompileRequest(this.entryPoint, super.completer); + final String entrypoint; + _CompileRequest(this.entrypoint, super.completer); } class _RecompileRequest extends _CompilationRequest { - final String entryPoint; + final String entrypoint; final List invalidatedFiles; - _RecompileRequest(this.entryPoint, this.invalidatedFiles, super.completer); + _RecompileRequest(this.entrypoint, this.invalidatedFiles, super.completer); } /// A single instance of the Frontend Server that persists across @@ -202,19 +208,19 @@ class PersistentFrontendServer { ); } - Future compile(String entryPoint) { + Future compile(String entrypoint) { _stdoutHandler.reset(); - _stdinController.add('compile $entryPoint'); + _stdinController.add('compile $entrypoint'); return _stdoutHandler.compilerOutput!.future; } Future recompile( - String entryPoint, + String entrypoint, List invalidatedFiles, ) { _stdoutHandler.reset(); final inputKey = const Uuid().v4(); - _stdinController.add('recompile $entryPoint $inputKey'); + _stdinController.add('recompile $entrypoint $inputKey'); for (final file in invalidatedFiles) { _stdinController.add(file.toString()); } @@ -226,13 +232,8 @@ class PersistentFrontendServer { _stdinController.add('accept'); } - void recordCompilerOutput(CompilerOutput output) { - if (output.errorCount != 0 || output.errorMessage != null) { - throw StateError( - 'Attempting to record compiler output with errors: ' - '${output.errorMessage}', - ); - } + /// Records all modified files into the in-memory filesystem. + void recordFiles() { final outputDillPath = outputDillUri.toFilePath(); final codeFile = File('$outputDillPath.sources'); final manifestFile = File('$outputDillPath.json'); @@ -248,6 +249,10 @@ class PersistentFrontendServer { _fileSystem.writeToDisk(_fileSystem.jsRootUri); } + void writeFile(String fileName) { + _fileSystem.writeFileToDisk(_fileSystem.jsRootUri, fileName); + } + Future shutdown() async { _stdinController.add('quit'); await _stdinController.close(); @@ -290,16 +295,57 @@ class WebMemoryFilesystem { WebMemoryFilesystem(this.jsRootUri); + /// Clears all files registered in the filesystem. + void clearWritableState() { + files.clear(); + sourcemaps.clear(); + metadata.clear(); + } + /// Writes the entirety of this filesystem to [outputDirectoryUri]. - /// - /// [clearWritableState] Should only be set on a reload or restart. - void writeToDisk(Uri outputDirectoryUri, {bool clearWritableState = false}) { + void writeToDisk(Uri outputDirectoryUri) { assert( Directory.fromUri(outputDirectoryUri).existsSync(), '$outputDirectoryUri does not exist.', ); var filesToWrite = {...files, ...sourcemaps, ...metadata}; - filesToWrite.forEach((path, content) { + _writeToDisk(outputDirectoryUri, filesToWrite); + } + + /// Writes [fileName] and its associated sourcemap and metadata files to + /// [outputDirectoryUri]. + void writeFileToDisk(Uri outputDirectoryUri, String fileName) { + assert( + Directory.fromUri(outputDirectoryUri).existsSync(), + '$outputDirectoryUri does not exist.', + ); + var sourceFile = fileName; + if (sourceFile.startsWith('package:')) { + sourceFile = + 'packages/${sourceFile.substring('package:'.length, sourceFile.length)}'; + } else if (sourceFile.startsWith('$multiRootScheme:///')) { + sourceFile = sourceFile.substring( + '$multiRootScheme:///'.length, + sourceFile.length, + ); + } + var sourceMapFile = '$sourceFile.map'; + var metadataFile = '$sourceFile.metadata'; + var filesToWrite = { + sourceFile: files[sourceFile]!, + sourceMapFile: sourcemaps[sourceMapFile]!, + metadataFile: metadata[metadataFile]!, + }; + + _writeToDisk(outputDirectoryUri, filesToWrite); + } + + /// Writes [rawFilesToWrite] to [outputDirectoryUri]. + void _writeToDisk( + Uri outputDirectoryUri, + Map rawFilesToWrite, + ) { + rawFilesToWrite.forEach((path, content) { final outputFileUri = outputDirectoryUri.resolve(path); var outputFilePath = outputFileUri.toFilePath().replaceFirst( '.dart.lib.js', @@ -309,10 +355,6 @@ class WebMemoryFilesystem { outputFile.createSync(recursive: true); outputFile.writeAsBytesSync(content); }); - if (clearWritableState) { - files.clear(); - sourcemaps.clear(); - } } /// Update the filesystem with the provided source and manifest files. diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 8b98dd3703..1e90f6f1f2 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -66,17 +66,13 @@ class DdcFrontendServerBuilder implements Builder { () => module.computeTransitiveAssets(buildStep), ); final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); - // Resolve the 'real' entrypoint we'll pass to FES for compilation. - final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); await buildStep.trackStage( 'EnsureAssets', - () => scratchSpace.ensureAssets([ - ...transitiveAssets, - webEntrypointAsset, - ], buildStep), + () => scratchSpace.ensureAssets(transitiveAssets, buildStep), ); var ddcEntrypointId = module.primarySource; var jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); + var jsFESOutputId = ddcEntrypointId.changeExtension('.dart.lib.js'); final frontendServer = await buildStep.fetchResource( persistentFrontendServerResource, @@ -85,13 +81,18 @@ class DdcFrontendServerBuilder implements Builder { frontendServerProxyDriverResource, ); driver.init(frontendServer); - final invalidatedFileUris = [for (var source in module.sources) source.uri]; + + // Request from the Frontend Server exactly the JS file requested by + // build_runner. Frontend Server's recompilation logic will avoid + // extraneous recompilation. + var invalidatedFiles = [ddcEntrypointId.uri]; await driver.recompileAndRecord( sourceArg(ddcEntrypointId), - invalidatedFileUris, + invalidatedFiles, + [sourceArg(jsFESOutputId)], ); final outputFile = scratchSpace.fileFor(jsOutputId); - // Write an empty file if this file was deemed extraneous by FES. + // Write an empty file if this output was deemed extraneous by FES. if (!!await outputFile.exists()) { await outputFile.create(recursive: true); } From efec641b83c4399591f387e4a6f91ded7a5b17fc Mon Sep 17 00:00:00 2001 From: MarkZ Date: Mon, 22 Sep 2025 19:32:15 -0700 Subject: [PATCH 05/19] Sample change with a hacky shared repo + digests impl comparisons --- build_runner/lib/build_runner.dart | 1 + build_runner/lib/src/build/build.dart | 17 ++++++ build_runner/lib/src/constants.dart | 11 +++- .../lib/src/ddc_frontend_server_builder.dart | 59 ++++++++++++++----- build_web_compilers/pubspec.yaml | 1 + scratch_space/lib/src/scratch_space.dart | 9 ++- 6 files changed, 81 insertions(+), 17 deletions(-) diff --git a/build_runner/lib/build_runner.dart b/build_runner/lib/build_runner.dart index e9fa775c62..ebf07ddddd 100644 --- a/build_runner/lib/build_runner.dart +++ b/build_runner/lib/build_runner.dart @@ -6,6 +6,7 @@ import 'src/build_plan/builder_factories.dart'; import 'src/build_runner.dart' show BuildRunner; export 'src/commands/daemon/constants.dart' show assetServerPort; +export 'src/constants.dart' show sharedBuildResourcesDirPath; /// Runs `build_runner` with [arguments] and [builderFactories]. /// diff --git a/build_runner/lib/src/build/build.dart b/build_runner/lib/src/build/build.dart index 43e6b4b04b..5fed082713 100644 --- a/build_runner/lib/src/build/build.dart +++ b/build_runner/lib/src/build/build.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'package:analyzer/dart/analysis/utilities.dart'; import 'package:analyzer/dart/ast/ast.dart'; @@ -225,6 +226,22 @@ class Build { final done = Completer(); runZonedGuarded( () async { + final sharedBuildResourcesDir = await Directory( + sharedBuildResourcesDirPath, + ).create(recursive: true); + final currentSharedBuildResourcesDir = await sharedBuildResourcesDir + .createTemp('shared'); + final currentBuildUpdatesFile = File.fromUri( + currentSharedBuildResourcesDir.uri.resolve('updates'), + ); + await currentBuildUpdatesFile.create(); + + await currentBuildUpdatesFile.writeAsString( + json.encode({ + for (final MapEntry(:key, :value) in updates.entries) + '${key.uri}': value.toString(), + }), + ); buildLog.doing('Updating the asset graph.'); if (!assetGraph.cleanBuild) { await _updateAssetGraph(updates); diff --git a/build_runner/lib/src/constants.dart b/build_runner/lib/src/constants.dart index c2c690b44c..7fac4a4ba1 100644 --- a/build_runner/lib/src/constants.dart +++ b/build_runner/lib/src/constants.dart @@ -11,6 +11,15 @@ import 'package:path/path.dart' as p; /// Relative path to the asset graph from the root package dir. final String assetGraphPath = assetGraphPathFor(_scriptPath); +/// Relative path to the directory containing shared resources for this build +/// (from the root package dir). +final String sharedBuildResourcesDirPath = sharedBuildResourcesDirFor( + _scriptPath, +); + +String sharedBuildResourcesDirFor(String path) => + '$cacheDir/${_scriptHashFor(path)}/shared/'; + /// Relative path to the asset graph for a build script at [path] String assetGraphPathFor(String path) => '$cacheDir/${_scriptHashFor(path)}/asset_graph.json'; @@ -25,7 +34,7 @@ final String _scriptPath = /// Directory containing automatically generated build entrypoints. /// /// Files in this directory must be read to do build script invalidation. -const entryPointDir = '$cacheDir/entrypoint'; +const String entryPointDir = '$cacheDir/entrypoint'; /// The directory to which hidden assets will be written. String get generatedOutputDirectory => '$cacheDir/generated'; diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 1e90f6f1f2..032af8c33e 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -4,9 +4,11 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; +import 'package:build_runner/build_runner.dart'; import 'common.dart'; import 'errors.dart'; @@ -30,11 +32,11 @@ class DdcFrontendServerBuilder implements Builder { @override Future build(BuildStep buildStep) async { - var moduleContents = await buildStep.readAsString(buildStep.inputId); + final moduleContents = await buildStep.readAsString(buildStep.inputId); final module = Module.fromJson( json.decode(moduleContents) as Map, ); - var ddcEntrypointId = module.primarySource; + final ddcEntrypointId = module.primarySource; // Entrypoints always have a `.module` file for ease of looking them up, // but they might not be the primary source. if (ddcEntrypointId.changeExtension(moduleExtension(ddcPlatform)) != @@ -61,18 +63,23 @@ class DdcFrontendServerBuilder implements Builder { /// Compile [module] with Frontend Server. Future _compile(Module module, BuildStep buildStep) async { - var transitiveAssets = await buildStep.trackStage( + final transitiveAssets = await buildStep.trackStage( 'CollectTransitiveDeps', () => module.computeTransitiveAssets(buildStep), ); final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); - await buildStep.trackStage( + final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); + final changedAssets = await buildStep.trackStage( 'EnsureAssets', - () => scratchSpace.ensureAssets(transitiveAssets, buildStep), + () => scratchSpace.ensureAssets([ + webEntrypointAsset, + ...transitiveAssets, + ], buildStep), ); - var ddcEntrypointId = module.primarySource; - var jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); - var jsFESOutputId = ddcEntrypointId.changeExtension('.dart.lib.js'); + final changedAssetUris = [for (final asset in changedAssets) asset.uri]; + final ddcEntrypointId = module.primarySource; + final jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); + final jsFESOutputId = ddcEntrypointId.changeExtension('.dart.lib.js'); final frontendServer = await buildStep.fetchResource( persistentFrontendServerResource, @@ -82,12 +89,36 @@ class DdcFrontendServerBuilder implements Builder { ); driver.init(frontendServer); + final sharedBuildResourcesDir = Directory(sharedBuildResourcesDirPath); + if (!sharedBuildResourcesDir.existsSync()) { + throw StateError( + 'Unable to read updated assets from $sharedBuildResourcesDir', + ); + } + final changedAssetsUrisFromFile = []; + for (final entity in sharedBuildResourcesDir.listSync(recursive: true)) { + if (entity is File) { + final updatedAssetsJson = + jsonDecode(entity.readAsStringSync()) as Map; + final currentUpdatedAssetsUris = [ + for (final entry in updatedAssetsJson.entries) Uri.parse(entry.key), + ]; + changedAssetsUrisFromFile.addAll(currentUpdatedAssetsUris); + } + } + + print('updatedAssetsUris $changedAssetsUrisFromFile'); + print('changedAssetUris $changedAssetUris'); + // Request from the Frontend Server exactly the JS file requested by // build_runner. Frontend Server's recompilation logic will avoid // extraneous recompilation. - var invalidatedFiles = [ddcEntrypointId.uri]; + final invalidatedFiles = [ + ddcEntrypointId.uri, + ...changedAssetsUrisFromFile, + ]; await driver.recompileAndRecord( - sourceArg(ddcEntrypointId), + sourceArg(webEntrypointAsset), invalidatedFiles, [sourceArg(jsFESOutputId)], ); @@ -105,10 +136,10 @@ class DdcFrontendServerBuilder implements Builder { // Copy the metadata output, modifying its contents to remove the temp // directory from paths - var metadataId = ddcEntrypointId.changeExtension(metadataExtension); - var file = scratchSpace.fileFor(metadataId); - var content = await file.readAsString(); - var json = jsonDecode(content) as Map; + final metadataId = ddcEntrypointId.changeExtension(metadataExtension); + final file = scratchSpace.fileFor(metadataId); + final content = await file.readAsString(); + final json = jsonDecode(content) as Map; fixMetadataSources(json, scratchSpace.tempDir.uri); await buildStep.writeAsString(metadataId, jsonEncode(json)); } diff --git a/build_web_compilers/pubspec.yaml b/build_web_compilers/pubspec.yaml index 99ac045fd5..16bdf59daf 100644 --- a/build_web_compilers/pubspec.yaml +++ b/build_web_compilers/pubspec.yaml @@ -13,6 +13,7 @@ dependencies: bazel_worker: ^1.0.0 build: '>=2.0.0 <5.0.0' build_modules: ^5.0.0 + build_runner: ^2.0.0 collection: ^1.15.0 glob: ^2.0.0 logging: ^1.0.0 diff --git a/scratch_space/lib/src/scratch_space.dart b/scratch_space/lib/src/scratch_space.dart index cf76e4bab6..cd17a9951c 100644 --- a/scratch_space/lib/src/scratch_space.dart +++ b/scratch_space/lib/src/scratch_space.dart @@ -98,10 +98,14 @@ class ScratchSpace { /// Any asset that is under a `lib` dir will be output under a `packages` /// directory corresponding to its package, and any other assets are output /// directly under the temp dir using their unmodified path. - Future ensureAssets(Iterable assetIds, AssetReader reader) { + Future> ensureAssets( + Iterable assetIds, + AssetReader reader, + ) { if (!exists) { throw StateError('Tried to use a deleted ScratchSpace!'); } + final result = {}; final futures = assetIds.map((id) async { @@ -112,6 +116,7 @@ class ScratchSpace { return; } _digests[id] = digest; + result.add(id); try { await _pendingWrites.putIfAbsent( @@ -131,7 +136,7 @@ class ScratchSpace { } }).toList(); - return Future.wait(futures); + return Future.wait(futures).then>((_) => result); } /// Returns the actual [File] in this environment corresponding to [id]. From aae24a2c66c6e05f6bda0e59bfcf1da12e6120c1 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 26 Sep 2025 12:19:35 -0700 Subject: [PATCH 06/19] Moving shared state to scratch space --- .../lib/src/frontend_server_driver.dart | 18 ++++++- build_modules/lib/src/module_builder.dart | 51 ++++++++++++------- build_modules/lib/src/scratch_space.dart | 3 ++ build_web_compilers/lib/builders.dart | 2 +- .../lib/src/ddc_frontend_server_builder.dart | 25 ++------- .../lib/src/web_entrypoint_builder.dart | 2 +- .../test/dev_compiler_builder_test.dart | 1 + scratch_space/lib/src/scratch_space.dart | 29 ++++++++--- 8 files changed, 81 insertions(+), 50 deletions(-) diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index 812750a218..a98dcd23e0 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -97,6 +97,7 @@ class FrontendServerProxyDriver { _cachedOutput = output = await _frontendServer!.compile(request.entrypoint); } else if (request is _RecompileRequest) { + print("PROCESSING ${request.entrypoint} ${request.invalidatedFiles}"); // Compile the first [_RecompileRequest] as a [_CompileRequest] to warm // up the Frontend Server. if (_cachedOutput == null) { @@ -113,7 +114,12 @@ class FrontendServerProxyDriver { } if (output != null && output.errorCount == 0) { _frontendServer!.accept(); + } else { + // We must await [reject]'s output, but we swallow the output since it + // doesn't provide useful information. + await _frontendServer!.reject(); } + print('OUTCOME ${output?.errorCount} ${output?.errorMessage}'); request.completer.complete(output); } catch (e, s) { request.completer.completeError(e, s); @@ -209,11 +215,13 @@ class PersistentFrontendServer { } Future compile(String entrypoint) { + print("COMPILE"); _stdoutHandler.reset(); _stdinController.add('compile $entrypoint'); return _stdoutHandler.compilerOutput!.future; } + /// Either [accept] or [reject] should be called after every [recompile] call. Future recompile( String entrypoint, List invalidatedFiles, @@ -232,6 +240,12 @@ class PersistentFrontendServer { _stdinController.add('accept'); } + Future reject() { + _stdoutHandler.reset(expectSources: false); + _stdinController.add('reject'); + return _stdoutHandler.compilerOutput!.future; + } + /// Records all modified files into the in-memory filesystem. void recordFiles() { final outputDillPath = outputDillUri.toFilePath(); @@ -505,6 +519,7 @@ class StdoutHandler { var _errorBuffer = StringBuffer(); void handler(String message) { + print("FES $message"); const kResultPrefix = 'result '; if (boundaryKey == null && message.startsWith(kResultPrefix)) { boundaryKey = message.substring(kResultPrefix.length); @@ -546,7 +561,7 @@ class StdoutHandler { _logger.info(message); _errorBuffer.writeln(message); case StdoutState.CollectDiagnostic: - _logger.info(message); + _logger.warning(message); _errorBuffer.writeln(message); case StdoutState.CollectDependencies: switch (message[0]) { @@ -565,7 +580,6 @@ class StdoutHandler { void reset({ bool suppressCompilerMessages = false, bool expectSources = true, - bool readFile = false, }) { boundaryKey = null; compilerOutput = Completer(); diff --git a/build_modules/lib/src/module_builder.dart b/build_modules/lib/src/module_builder.dart index 5b9caa3447..0e149d9568 100644 --- a/build_modules/lib/src/module_builder.dart +++ b/build_modules/lib/src/module_builder.dart @@ -14,6 +14,7 @@ import 'module_library.dart'; import 'module_library_builder.dart' show moduleLibraryExtension; import 'modules.dart'; import 'platform.dart'; +import 'scratch_space.dart'; /// The extension for serialized module assets. String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; @@ -23,13 +24,15 @@ String moduleExtension(DartPlatform platform) => '.${platform.name}.module'; class ModuleBuilder implements Builder { final DartPlatform _platform; - /// True if we use raw modules instead of clean modules. + /// Emits DDC code with the Library Bundle module system, which supports hot + /// reload. /// + /// If set, this builder will consume raw meta modules (instead of clean). /// Clean meta modules are only used for DDC's AMD module system due its /// requirement that self-referential libraries be bundled. - final bool useRawMetaModules; + final bool usesWebHotReload; - ModuleBuilder(this._platform, {this.useRawMetaModules = false}) + ModuleBuilder(this._platform, {this.usesWebHotReload = false}) : buildExtensions = { '.dart': [moduleExtension(_platform)], }; @@ -40,8 +43,8 @@ class ModuleBuilder implements Builder { @override Future build(BuildStep buildStep) async { final metaModules = await buildStep.fetchResource(metaModuleCache); - var metaModuleExtensionString = - useRawMetaModules + final metaModuleExtensionString = + usesWebHotReload ? metaModuleExtension(_platform) : metaModuleCleanExtension(_platform); final metaModule = @@ -52,19 +55,25 @@ class ModuleBuilder implements Builder { var outputModule = metaModule.modules.firstWhereOrNull( (m) => m.primarySource == buildStep.inputId, ); - if (outputModule == null) { - final serializedLibrary = await buildStep.readAsString( - buildStep.inputId.changeExtension(moduleLibraryExtension), - ); - final libraryModule = ModuleLibrary.deserialize( - buildStep.inputId, - serializedLibrary, + final serializedLibrary = await buildStep.readAsString( + buildStep.inputId.changeExtension(moduleLibraryExtension), + ); + final libraryModule = ModuleLibrary.deserialize( + buildStep.inputId, + serializedLibrary, + ); + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + if (usesWebHotReload && + libraryModule.isEntryPoint && + libraryModule.hasMain) { + // We must save the main entrypoint as the recompilation target for the + // Frontend Server before any JS files are emitted. + scratchSpace.entrypointAssetId = libraryModule.id; + } + if (outputModule == null && libraryModule.hasMain) { + outputModule = metaModule.modules.firstWhere( + (m) => m.sources.contains(buildStep.inputId), ); - if (libraryModule.hasMain) { - outputModule = metaModule.modules.firstWhere( - (m) => m.sources.contains(buildStep.inputId), - ); - } } if (outputModule == null) return; final modules = await buildStep.fetchResource(moduleCache); @@ -73,5 +82,13 @@ class ModuleBuilder implements Builder { buildStep, outputModule, ); + if (usesWebHotReload) { + // All sources must be declared before the Frontend Server is invoked, as + // it only accepts the main entrypoint as its compilation target. + await buildStep.trackStage( + 'EnsureAssets', + () => scratchSpace.ensureAssets(outputModule!.sources, buildStep), + ); + } } } diff --git a/build_modules/lib/src/scratch_space.dart b/build_modules/lib/src/scratch_space.dart index e0d17021f2..6c8bf35f32 100644 --- a/build_modules/lib/src/scratch_space.dart +++ b/build_modules/lib/src/scratch_space.dart @@ -44,6 +44,9 @@ final scratchSpaceResource = Resource( } return scratchSpace; }, + dispose: (scratchSpace) { + scratchSpace.dispose(); + }, beforeExit: () async { // The workers are running inside the scratch space, so wait for them to // shut down before deleting it. diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index 8c255e248e..b2156c0226 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -33,7 +33,7 @@ Builder ddcModuleBuilder(BuilderOptions options) { _ensureSameDdcHotReloadOptions(options); return ModuleBuilder( ddcPlatform, - useRawMetaModules: _readWebHotReloadOption(options), + usesWebHotReload: _readWebHotReloadOption(options), ); } diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 032af8c33e..0011ee39b4 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -68,14 +68,15 @@ class DdcFrontendServerBuilder implements Builder { () => module.computeTransitiveAssets(buildStep), ); final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); - final webEntrypointAsset = AssetId.resolve(Uri.parse(entrypoint)); - final changedAssets = await buildStep.trackStage( + final webEntrypointAsset = scratchSpace.entrypointAssetId; + await buildStep.trackStage( 'EnsureAssets', () => scratchSpace.ensureAssets([ webEntrypointAsset, ...transitiveAssets, ], buildStep), ); + final changedAssets = scratchSpace.changedFilesInBuild; final changedAssetUris = [for (final asset in changedAssets) asset.uri]; final ddcEntrypointId = module.primarySource; final jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); @@ -95,31 +96,13 @@ class DdcFrontendServerBuilder implements Builder { 'Unable to read updated assets from $sharedBuildResourcesDir', ); } - final changedAssetsUrisFromFile = []; - for (final entity in sharedBuildResourcesDir.listSync(recursive: true)) { - if (entity is File) { - final updatedAssetsJson = - jsonDecode(entity.readAsStringSync()) as Map; - final currentUpdatedAssetsUris = [ - for (final entry in updatedAssetsJson.entries) Uri.parse(entry.key), - ]; - changedAssetsUrisFromFile.addAll(currentUpdatedAssetsUris); - } - } - - print('updatedAssetsUris $changedAssetsUrisFromFile'); - print('changedAssetUris $changedAssetUris'); // Request from the Frontend Server exactly the JS file requested by // build_runner. Frontend Server's recompilation logic will avoid // extraneous recompilation. - final invalidatedFiles = [ - ddcEntrypointId.uri, - ...changedAssetsUrisFromFile, - ]; await driver.recompileAndRecord( sourceArg(webEntrypointAsset), - invalidatedFiles, + changedAssetUris, [sourceArg(jsFESOutputId)], ); final outputFile = scratchSpace.fileFor(jsOutputId); diff --git a/build_web_compilers/lib/src/web_entrypoint_builder.dart b/build_web_compilers/lib/src/web_entrypoint_builder.dart index cd82aec1ac..11bbdd8c45 100644 --- a/build_web_compilers/lib/src/web_entrypoint_builder.dart +++ b/build_web_compilers/lib/src/web_entrypoint_builder.dart @@ -327,7 +327,7 @@ class WebEntrypointBuilder implements Builder { compilationSteps.add( Future(() async { try { - var usesWebHotReload = options.usesWebHotReload; + final usesWebHotReload = options.usesWebHotReload; await bootstrapDdc( buildStep, nativeNullAssertions: options.nativeNullAssertions, diff --git a/build_web_compilers/test/dev_compiler_builder_test.dart b/build_web_compilers/test/dev_compiler_builder_test.dart index e3e7f6fe14..6eab686d37 100644 --- a/build_web_compilers/test/dev_compiler_builder_test.dart +++ b/build_web_compilers/test/dev_compiler_builder_test.dart @@ -7,6 +7,7 @@ import 'package:build_modules/build_modules.dart'; import 'package:build_test/build_test.dart'; import 'package:build_web_compilers/build_web_compilers.dart'; import 'package:build_web_compilers/builders.dart'; +import 'package:build_web_compilers/src/common.dart'; import 'package:logging/logging.dart'; import 'package:test/test.dart'; diff --git a/scratch_space/lib/src/scratch_space.dart b/scratch_space/lib/src/scratch_space.dart index cd17a9951c..6bcc69be68 100644 --- a/scratch_space/lib/src/scratch_space.dart +++ b/scratch_space/lib/src/scratch_space.dart @@ -21,12 +21,21 @@ class ScratchSpace { /// Whether or not this scratch space still exists. bool exists = true; + /// The built app's main entrypoint file. + /// + /// This must be set before any asset builders run when compiling with DDC and + /// hot reload. + late AssetId entrypointAssetId; + /// The `packages` directory under the temp directory. final Directory packagesDir; /// The temp directory at the root of this [ScratchSpace]. final Directory tempDir; + /// Holds all files that have been locally modified in this build. + final changedFilesInBuild = {}; + // Assets which have a file created but are still being written to. final _pendingWrites = >{}; @@ -92,21 +101,18 @@ class ScratchSpace { /// Copies [assetIds] to [tempDir] if they don't exist, using [reader] to /// read assets and mark dependencies. /// + /// Locally updated assets will be recorded in [changedFilesInBuild]. + /// /// Note that [BuildStep] implements [AssetReader] and that is typically /// what you will want to pass in. /// /// Any asset that is under a `lib` dir will be output under a `packages` /// directory corresponding to its package, and any other assets are output /// directly under the temp dir using their unmodified path. - Future> ensureAssets( - Iterable assetIds, - AssetReader reader, - ) { + Future ensureAssets(Iterable assetIds, AssetReader reader) { if (!exists) { throw StateError('Tried to use a deleted ScratchSpace!'); } - final result = {}; - final futures = assetIds.map((id) async { final digest = await reader.digest(id); @@ -115,8 +121,10 @@ class ScratchSpace { await _pendingWrites[id]; return; } + if (existing != null) { + changedFilesInBuild.add(id); + } _digests[id] = digest; - result.add(id); try { await _pendingWrites.putIfAbsent( @@ -136,7 +144,7 @@ class ScratchSpace { } }).toList(); - return Future.wait(futures).then>((_) => result); + return Future.wait(futures); } /// Returns the actual [File] in this environment corresponding to [id]. @@ -145,6 +153,11 @@ class ScratchSpace { /// with [id] to make sure it is actually present. File fileFor(AssetId id) => File(p.join(tempDir.path, p.normalize(_relativePathFor(id)))); + + /// Performs cleanup required across builds. + void dispose() { + changedFilesInBuild.clear(); + } } /// Returns a canonical uri for [id]. From 51f1e0d3f0a0a7afa98ab453f0c4b512e32c3c53 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Mon, 29 Sep 2025 12:23:14 -0700 Subject: [PATCH 07/19] addressing analysis errors --- .../lib/src/frontend_server_driver.dart | 27 +++++++++---------- .../test/meta_module_builder_test.dart | 1 - build_modules/test/meta_module_test.dart | 1 - build_modules/test/module_builder_test.dart | 1 - .../lib/build_web_compilers.dart | 4 +-- build_web_compilers/lib/builders.dart | 2 +- .../lib/src/ddc_frontend_server_builder.dart | 2 -- .../lib/src/dev_compiler_bootstrap.dart | 4 +-- .../lib/src/dev_compiler_builder.dart | 11 ++++---- build_web_compilers/pubspec.yaml | 1 - .../test/dev_compiler_builder_test.dart | 1 - 11 files changed, 23 insertions(+), 32 deletions(-) diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index a98dcd23e0..00c9edc303 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -50,7 +50,7 @@ class FrontendServerProxyDriver { List invalidatedFiles, Iterable filesToWrite, ) async { - var compilerOutput = await recompile(entrypoint, invalidatedFiles); + final compilerOutput = await recompile(entrypoint, invalidatedFiles); if (compilerOutput == null) { throw Exception('Frontend Server failed to recompile $entrypoint'); } @@ -61,7 +61,7 @@ class FrontendServerProxyDriver { ); } _frontendServer!.recordFiles(); - for (var file in filesToWrite) { + for (final file in filesToWrite) { _frontendServer!.writeFile(file); } return compilerOutput; @@ -97,7 +97,6 @@ class FrontendServerProxyDriver { _cachedOutput = output = await _frontendServer!.compile(request.entrypoint); } else if (request is _RecompileRequest) { - print("PROCESSING ${request.entrypoint} ${request.invalidatedFiles}"); // Compile the first [_RecompileRequest] as a [_CompileRequest] to warm // up the Frontend Server. if (_cachedOutput == null) { @@ -119,7 +118,6 @@ class FrontendServerProxyDriver { // doesn't provide useful information. await _frontendServer!.reject(); } - print('OUTCOME ${output?.errorCount} ${output?.errorMessage}'); request.completer.complete(output); } catch (e, s) { request.completer.completeError(e, s); @@ -194,7 +192,7 @@ class PersistentFrontendServer { '--output-incremental-dill=${outputDillUri.toFilePath()}', ]; final process = await Process.start(dartaotruntimePath, args); - var fileSystem = WebMemoryFilesystem(fileSystemRoot); + final fileSystem = WebMemoryFilesystem(fileSystemRoot); final stdoutHandler = StdoutHandler(logger: _log); process.stdout .transform(utf8.decoder) @@ -215,7 +213,6 @@ class PersistentFrontendServer { } Future compile(String entrypoint) { - print("COMPILE"); _stdoutHandler.reset(); _stdinController.add('compile $entrypoint'); return _stdoutHandler.compilerOutput!.future; @@ -322,7 +319,7 @@ class WebMemoryFilesystem { Directory.fromUri(outputDirectoryUri).existsSync(), '$outputDirectoryUri does not exist.', ); - var filesToWrite = {...files, ...sourcemaps, ...metadata}; + final filesToWrite = {...files, ...sourcemaps, ...metadata}; _writeToDisk(outputDirectoryUri, filesToWrite); } @@ -343,9 +340,9 @@ class WebMemoryFilesystem { sourceFile.length, ); } - var sourceMapFile = '$sourceFile.map'; - var metadataFile = '$sourceFile.metadata'; - var filesToWrite = { + final sourceMapFile = '$sourceFile.map'; + final metadataFile = '$sourceFile.metadata'; + final filesToWrite = { sourceFile: files[sourceFile]!, sourceMapFile: sourcemaps[sourceMapFile]!, metadataFile: metadata[metadataFile]!, @@ -361,7 +358,7 @@ class WebMemoryFilesystem { ) { rawFilesToWrite.forEach((path, content) { final outputFileUri = outputDirectoryUri.resolve(path); - var outputFilePath = outputFileUri.toFilePath().replaceFirst( + final outputFilePath = outputFileUri.toFilePath().replaceFirst( '.dart.lib.js', '.ddc.js', ); @@ -437,8 +434,11 @@ class WebMemoryFilesystem { // from JS files and might not generalize. var libraryName = dartFileName; if (libraryName.startsWith('packages/')) { - libraryName = - 'package:${libraryName.substring('packages/'.length, libraryName.length)}'; + final libraryNameWithoutPrefix = libraryName.substring( + 'packages/'.length, + libraryName.length, + ); + libraryName = 'package:$libraryNameWithoutPrefix'; } else { libraryName = '$multiRootScheme:///$libraryName'; } @@ -519,7 +519,6 @@ class StdoutHandler { var _errorBuffer = StringBuffer(); void handler(String message) { - print("FES $message"); const kResultPrefix = 'result '; if (boundaryKey == null && message.startsWith(kResultPrefix)) { boundaryKey = message.substring(kResultPrefix.length); diff --git a/build_modules/test/meta_module_builder_test.dart b/build_modules/test/meta_module_builder_test.dart index 56b11dc779..c208399ee6 100644 --- a/build_modules/test/meta_module_builder_test.dart +++ b/build_modules/test/meta_module_builder_test.dart @@ -5,7 +5,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_modules/test/meta_module_test.dart b/build_modules/test/meta_module_test.dart index 1917d48e54..5b0aa9ce34 100644 --- a/build_modules/test/meta_module_test.dart +++ b/build_modules/test/meta_module_test.dart @@ -8,7 +8,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/common.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_modules/test/module_builder_test.dart b/build_modules/test/module_builder_test.dart index 065ffd9b05..095756e7a5 100644 --- a/build_modules/test/module_builder_test.dart +++ b/build_modules/test/module_builder_test.dart @@ -7,7 +7,6 @@ import 'dart:convert'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'package:build_modules/src/meta_module.dart'; -import 'package:build_modules/src/module_library.dart'; import 'package:build_test/build_test.dart'; import 'package:test/test.dart'; diff --git a/build_web_compilers/lib/build_web_compilers.dart b/build_web_compilers/lib/build_web_compilers.dart index 92e101ead4..8044292333 100644 --- a/build_web_compilers/lib/build_web_compilers.dart +++ b/build_web_compilers/lib/build_web_compilers.dart @@ -3,15 +3,15 @@ // BSD-style license that can be found in the LICENSE file. export 'src/archive_extractor.dart' show Dart2JsArchiveExtractor; -export 'src/dev_compiler_builder.dart' +export 'src/common.dart' show - DevCompilerBuilder, fullKernelExtension, jsModuleErrorsExtension, jsModuleExtension, jsSourceMapExtension, metadataExtension, symbolsExtension; +export 'src/dev_compiler_builder.dart' show DevCompilerBuilder; export 'src/platforms.dart' show dart2jsPlatform, dart2wasmPlatform, ddcPlatform; export 'src/web_entrypoint_builder.dart' diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index b2156c0226..ad674dd8c9 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -150,7 +150,7 @@ void _ensureSameDdcOptions(BuilderOptions options) { } void _ensureSameDdcHotReloadOptions(BuilderOptions options) { - var webHotReload = _readWebHotReloadOption(options); + final webHotReload = _readWebHotReloadOption(options); if (_lastWebHotReloadValue != null) { if (webHotReload != _lastWebHotReloadValue) { throw ArgumentError( diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 019527986f..e2c04b4456 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -4,11 +4,9 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:io'; import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; -import 'package:build_runner/build_runner.dart'; import 'common.dart'; import 'errors.dart'; diff --git a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart index 968c41dbcc..b8982e13dd 100644 --- a/build_web_compilers/lib/src/dev_compiler_bootstrap.dart +++ b/build_web_compilers/lib/src/dev_compiler_bootstrap.dart @@ -687,13 +687,13 @@ String generateDDCLibraryBundleBootstrapScript({ required bool isWindows, required Map scriptIdsToPath, }) { - var scriptsJs = StringBuffer(); + final scriptsJs = StringBuffer(); scriptIdsToPath.forEach((id, path) { scriptsJs.write('{"src": "$path", "id": "$id"},\n'); }); // Write the "true" main boostrapper last as part of the loader's convention. scriptsJs.write('{"src": "$mainBoostrapperUrl", "id": "data-main"}\n'); - var boostrapScript = ''' + final boostrapScript = ''' // Save the current directory so we can access it in a closure. var _currentDirectory = (function () { var _url = document.currentScript.src; diff --git a/build_web_compilers/lib/src/dev_compiler_builder.dart b/build_web_compilers/lib/src/dev_compiler_builder.dart index 7850ac108e..66cbe235cd 100644 --- a/build_web_compilers/lib/src/dev_compiler_builder.dart +++ b/build_web_compilers/lib/src/dev_compiler_builder.dart @@ -11,7 +11,6 @@ import 'package:build/build.dart'; import 'package:build/experiments.dart'; import 'package:build_modules/build_modules.dart'; import 'package:path/path.dart' as p; -import 'package:scratch_space/scratch_space.dart'; import '../builders.dart'; import 'common.dart'; @@ -211,7 +210,7 @@ Future _createDevCompilerModule( '-o', jsOutputFile.path, debugMode ? '--source-map' : '--no-source-map', - for (var dep in transitiveDeps) _summaryArg(dep), + for (final dep in transitiveDeps) _summaryArg(dep), '--packages=$multiRootScheme:///.dart_tool/package_config.json', '--module-name=${ddcModuleName(jsId)}', '--multi-root-scheme=$multiRootScheme', @@ -226,10 +225,10 @@ Future _createDevCompilerModule( ], if (usedInputsFile != null) '--used-inputs-file=${usedInputsFile.uri.toFilePath()}', - for (var source in module.sources) sourceArg(source), - for (var define in environment.entries) + for (final source in module.sources) sourceArg(source), + for (final define in environment.entries) '-D${define.key}=${define.value}', - for (var experiment in enabledExperiments) + for (final experiment in enabledExperiments) '--enable-experiment=$experiment', ]) ..inputs.add( @@ -240,7 +239,7 @@ Future _createDevCompilerModule( ..inputs.addAll( await Future.wait( transitiveKernelDeps.map((dep) async { - var file = scratchSpace.fileFor(dep); + final file = scratchSpace.fileFor(dep); if (kernelInputPathToId != null) { kernelInputPathToId[file.uri.toString()] = dep; } diff --git a/build_web_compilers/pubspec.yaml b/build_web_compilers/pubspec.yaml index 16bdf59daf..9b6e1a5d25 100644 --- a/build_web_compilers/pubspec.yaml +++ b/build_web_compilers/pubspec.yaml @@ -29,7 +29,6 @@ dev_dependencies: path: ../build_modules/test/fixtures/a b: path: ../build_modules/test/fixtures/b - build_runner: ^2.0.0 build_test: ^3.0.0 c: path: test/fixtures/c diff --git a/build_web_compilers/test/dev_compiler_builder_test.dart b/build_web_compilers/test/dev_compiler_builder_test.dart index 6eab686d37..e3e7f6fe14 100644 --- a/build_web_compilers/test/dev_compiler_builder_test.dart +++ b/build_web_compilers/test/dev_compiler_builder_test.dart @@ -7,7 +7,6 @@ import 'package:build_modules/build_modules.dart'; import 'package:build_test/build_test.dart'; import 'package:build_web_compilers/build_web_compilers.dart'; import 'package:build_web_compilers/builders.dart'; -import 'package:build_web_compilers/src/common.dart'; import 'package:logging/logging.dart'; import 'package:test/test.dart'; From 5d309cabffc4f0ebc6cdf808533fa0dd3f116f92 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Mon, 29 Sep 2025 13:24:40 -0700 Subject: [PATCH 08/19] additional file cleanup --- build_modules/lib/src/ddc_names.dart | 2 +- build_modules/test/module_builder_test.dart | 52 +++++++++++++++++++ build_runner/lib/src/constants.dart | 2 +- .../lib/src/ddc_frontend_server_builder.dart | 7 +-- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/build_modules/lib/src/ddc_names.dart b/build_modules/lib/src/ddc_names.dart index f2ece5293a..fbcf80a10a 100644 --- a/build_modules/lib/src/ddc_names.dart +++ b/build_modules/lib/src/ddc_names.dart @@ -66,7 +66,7 @@ String toJSIdentifier(String name) { /// Also handles invalid variable names in strict mode, like "arguments". bool invalidVariableName(String keyword, {bool strictMode = true}) { switch (keyword) { - // https: //262.ecma-international.org/6.0/#sec-reserved-words + // https://262.ecma-international.org/6.0/#sec-reserved-words case 'true': case 'false': case 'null': diff --git a/build_modules/test/module_builder_test.dart b/build_modules/test/module_builder_test.dart index 095756e7a5..01a065e9b8 100644 --- a/build_modules/test/module_builder_test.dart +++ b/build_modules/test/module_builder_test.dart @@ -60,4 +60,56 @@ void main() { }, ); }); + + test('can serialize modules and output all modules when strongly connected ' + 'components are disabled', () async { + final assetA = AssetId('a', 'lib/a.dart'); + final assetB = AssetId('a', 'lib/b.dart'); + final assetC = AssetId('a', 'lib/c.dart'); + final assetD = AssetId('a', 'lib/d.dart'); + final assetE = AssetId('a', 'lib/e.dart'); + final moduleA = Module(assetA, [assetA], [], platform, true); + final moduleB = Module( + assetB, + [assetB, assetC], + [], + platform, + true, + ); + final moduleC = Module(assetC, [assetC], [], platform, true); + + final moduleD = Module( + assetD, + [assetD, assetE], + [], + platform, + false, + ); + final moduleE = Module(assetE, [assetE], [], platform, true); + final metaModule = MetaModule([moduleA, moduleB, moduleD]); + await testBuilder( + ModuleBuilder(platform, usesWebHotReload: true), + { + 'a|lib/a.dart': '', + 'a|lib/b.dart': '', + 'a|lib/c.dart': '', + 'a|lib/d.dart': '', + 'a|lib/e.dart': '', + 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( + metaModule.toJson(), + ), + 'a|lib/c$moduleLibraryExtension': + ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/e$moduleLibraryExtension': + ModuleLibrary.fromSource(assetE, '').serialize(), + }, + outputs: { + 'a|lib/a${moduleExtension(platform)}': encodedMatchesModule(moduleA), + 'a|lib/b${moduleExtension(platform)}': encodedMatchesModule(moduleB), + 'a|lib/c${moduleExtension(platform)}': encodedMatchesModule(moduleC), + 'a|lib/d${moduleExtension(platform)}': encodedMatchesModule(moduleD), + 'a|lib/e${moduleExtension(platform)}': encodedMatchesModule(moduleE), + }, + ); + }); } diff --git a/build_runner/lib/src/constants.dart b/build_runner/lib/src/constants.dart index ee19c4f0a1..055beec637 100644 --- a/build_runner/lib/src/constants.dart +++ b/build_runner/lib/src/constants.dart @@ -31,7 +31,7 @@ const String entryPointDir = '$cacheDir/entrypoint'; String get generatedOutputDirectory => '$cacheDir/generated'; /// Relative path to the cache directory from the root package dir. -const String cacheDir = '.dart_tool/build'; +const cacheDir = '.dart_tool/build'; /// Returns a hash for a given Dart script path. /// diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index e2c04b4456..ca8be764a1 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -12,7 +12,7 @@ import 'common.dart'; import 'errors.dart'; import 'platforms.dart'; -/// A builder which can compile DDC modules with the Frontend Server. +/// A builder that compiles DDC modules with the Frontend Server. class DdcFrontendServerBuilder implements Builder { final String entrypoint; @@ -74,8 +74,9 @@ class DdcFrontendServerBuilder implements Builder { ...transitiveAssets, ], buildStep), ); - final changedAssets = scratchSpace.changedFilesInBuild; - final changedAssetUris = [for (final asset in changedAssets) asset.uri]; + final changedAssetUris = [ + for (final asset in scratchSpace.changedFilesInBuild) asset.uri, + ]; final ddcEntrypointId = module.primarySource; final jsOutputId = ddcEntrypointId.changeExtension(jsModuleExtension); final jsFESOutputId = ddcEntrypointId.changeExtension('.dart.lib.js'); From 9a37369adf1a883804102290f64e283673c6ef4e Mon Sep 17 00:00:00 2001 From: MarkZ Date: Mon, 29 Sep 2025 16:18:59 -0700 Subject: [PATCH 09/19] Adding build_modules tests --- .../test/frontend_server_driver_test.dart | 174 ++++++++++++++++++ build_modules/test/module_builder_test.dart | 23 ++- 2 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 build_modules/test/frontend_server_driver_test.dart diff --git a/build_modules/test/frontend_server_driver_test.dart b/build_modules/test/frontend_server_driver_test.dart new file mode 100644 index 0000000000..e62bf6a5f4 --- /dev/null +++ b/build_modules/test/frontend_server_driver_test.dart @@ -0,0 +1,174 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:build_modules/src/common.dart'; +import 'package:build_modules/src/frontend_server_driver.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + group('Frontend Server', () { + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + }); + + tearDown(() async { + await server.shutdown(); + await tempDir.delete(recursive: true); + }); + + test('can compile a simple dart file', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + expect(output.outputFilename, endsWith('output.dill')); + expect(output.sources, contains(entrypoint)); + server.accept(); + }); + + test('can recompile with an invalidated file', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + final dep = tempDir.uri.resolve('dep.dart'); + File( + entrypoint.toFilePath(), + ).writeAsStringSync("import 'dep.dart'; void main() {}"); + File(dep.toFilePath()).writeAsStringSync('// empty'); + + var output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + + File(dep.toFilePath()).writeAsStringSync('invalid dart code'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, contains("Error: Expected ';' after this.")); + await server.reject(); + }); + + test('can handle compilation errors', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, isNotNull); + expect( + output.errorMessage, + contains("Error: Can't find '}' to match '{'"), + ); + await server.reject(); + }); + }); + + group('Frontend Server Proxy', () { + late FrontendServerProxyDriver driver; + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + driver = FrontendServerProxyDriver(); + driver.init(server); + }); + + tearDown(() async { + await driver.terminate(); + await tempDir.delete(recursive: true); + }); + + test('can recompile after valid edits', () async { + final file1 = tempDir.uri.resolve('file1.dart'); + File(file1.toFilePath()).writeAsStringSync('void main() {}'); + final file2 = tempDir.uri.resolve('file2.dart'); + File(file2.toFilePath()).writeAsStringSync('void main() {}'); + + var future1 = driver.recompile(file1.toString(), []); + var future2 = driver.recompile(file2.toString(), []); + + var results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + + File( + file1.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + File( + file2.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + + future1 = driver.recompile(file1.toString(), [file1]); + future2 = driver.recompile(file2.toString(), [file2]); + + results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + }); + + test('recompileAndRecord successfully records and writes files', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final jsFESOutputPath = p.join(tempDir.path, 'entrypoint.dart.lib.js'); + final output = await driver.recompileAndRecord( + '$multiRootScheme:///entrypoint.dart', + [entrypoint], + ['entrypoint.dart.lib.js'], + ); + + expect(output, isNotNull); + expect(output.errorCount, 0); + + final jsOutputFile = File( + jsFESOutputPath.replaceFirst('.dart.lib.js', '.ddc.js'), + ); + print(jsOutputFile.path); + expect(jsOutputFile.existsSync(), isTrue); + final content = jsOutputFile.readAsStringSync(); + expect(content, contains('function main()')); + }); + }); +} diff --git a/build_modules/test/module_builder_test.dart b/build_modules/test/module_builder_test.dart index 01a065e9b8..3a684b5cfe 100644 --- a/build_modules/test/module_builder_test.dart +++ b/build_modules/test/module_builder_test.dart @@ -48,8 +48,14 @@ void main() { 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( metaModule.toJson(), ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), 'a|lib/c$moduleLibraryExtension': ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), 'a|lib/e$moduleLibraryExtension': ModuleLibrary.fromSource(assetE, '').serialize(), }, @@ -77,7 +83,6 @@ void main() { true, ); final moduleC = Module(assetC, [assetC], [], platform, true); - final moduleD = Module( assetD, [assetD, assetE], @@ -86,7 +91,13 @@ void main() { false, ); final moduleE = Module(assetE, [assetE], [], platform, true); - final metaModule = MetaModule([moduleA, moduleB, moduleD]); + final metaModule = MetaModule([ + moduleA, + moduleB, + moduleC, + moduleD, + moduleE, + ]); await testBuilder( ModuleBuilder(platform, usesWebHotReload: true), { @@ -95,11 +106,17 @@ void main() { 'a|lib/c.dart': '', 'a|lib/d.dart': '', 'a|lib/e.dart': '', - 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( + 'a|lib/${metaModuleExtension(platform)}': jsonEncode( metaModule.toJson(), ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), 'a|lib/c$moduleLibraryExtension': ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), 'a|lib/e$moduleLibraryExtension': ModuleLibrary.fromSource(assetE, '').serialize(), }, From b2bab91501f6370ca9c95a15cc4bd7d6f95a13af Mon Sep 17 00:00:00 2001 From: MarkZ Date: Mon, 29 Sep 2025 16:18:59 -0700 Subject: [PATCH 10/19] Adding build_modules tests --- .../test/frontend_server_driver_test.dart | 179 ++++++++++++++++++ build_modules/test/module_builder_test.dart | 23 ++- 2 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 build_modules/test/frontend_server_driver_test.dart diff --git a/build_modules/test/frontend_server_driver_test.dart b/build_modules/test/frontend_server_driver_test.dart new file mode 100644 index 0000000000..7e37d0ee83 --- /dev/null +++ b/build_modules/test/frontend_server_driver_test.dart @@ -0,0 +1,179 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:build_modules/src/common.dart'; +import 'package:build_modules/src/frontend_server_driver.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + group('Frontend Server', () { + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + }); + + tearDown(() async { + await server.shutdown(); + await tempDir.delete(recursive: true); + }); + + test('can compile a simple dart file', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + expect(output.outputFilename, endsWith('output.dill')); + expect(output.sources, contains(entrypoint)); + server.accept(); + }); + + test('can handle compilation errors', () async { + final entrypoint = tempDir.uri.resolve('entry.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {'); + + final output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, isNotNull); + expect( + output.errorMessage, + contains("Error: Can't find '}' to match '{'"), + ); + await server.reject(); + }); + + test('can reject and recompile with an invalidated file', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + final dep = tempDir.uri.resolve('dep.dart'); + File( + entrypoint.toFilePath(), + ).writeAsStringSync("import 'dep.dart'; void main() {}"); + File(dep.toFilePath()).writeAsStringSync('// empty'); + + var output = await server.compile(entrypoint.toString()); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + + File(dep.toFilePath()).writeAsStringSync('invalid dart code'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, greaterThan(0)); + expect(output.errorMessage, contains("Error: Expected ';' after this.")); + await server.reject(); + + File(dep.toFilePath()).writeAsStringSync('// empty'); + output = await server.recompile(entrypoint.toString(), [dep]); + expect(output, isNotNull); + expect(output!.errorCount, 0); + server.accept(); + }); + }); + + group('Frontend Server Proxy', () { + late FrontendServerProxyDriver driver; + late PersistentFrontendServer server; + late Directory tempDir; + + setUp(() async { + tempDir = await Directory.systemTemp.createTemp('fes-test'); + final packageConfig = tempDir.uri.resolve( + '.dart_tool/package_config.json', + ); + File(packageConfig.toFilePath()) + ..createSync(recursive: true) + ..writeAsStringSync( + jsonEncode({'configVersion': 2, 'packages': []}), + ); + server = await PersistentFrontendServer.start( + sdkRoot: sdkDir, + fileSystemRoot: tempDir.uri, + packagesFile: packageConfig, + ); + driver = FrontendServerProxyDriver(); + driver.init(server); + }); + + tearDown(() async { + await driver.terminate(); + await tempDir.delete(recursive: true); + }); + + test('can recompile after valid edits', () async { + final file1 = tempDir.uri.resolve('file1.dart'); + File(file1.toFilePath()).writeAsStringSync('void main() {}'); + final file2 = tempDir.uri.resolve('file2.dart'); + File(file2.toFilePath()).writeAsStringSync('void main() {}'); + + var future1 = driver.recompile(file1.toString(), []); + var future2 = driver.recompile(file2.toString(), []); + + var results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + + File( + file1.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + File( + file2.toFilePath(), + ).writeAsStringSync("void main() { print('edit!'); }"); + + future1 = driver.recompile(file1.toString(), [file1]); + future2 = driver.recompile(file2.toString(), [file2]); + + results = await Future.wait([future1, future2]); + expect(results[0], isNotNull); + expect(results[0]!.errorCount, 0); + expect(results[1], isNotNull); + expect(results[1]!.errorCount, 0); + }); + + test('recompileAndRecord successfully records and writes files', () async { + final entrypoint = tempDir.uri.resolve('entrypoint.dart'); + File(entrypoint.toFilePath()).writeAsStringSync('void main() {}'); + + final jsFESOutputPath = p.join(tempDir.path, 'entrypoint.dart.lib.js'); + final output = await driver.recompileAndRecord( + '$multiRootScheme:///entrypoint.dart', + [entrypoint], + ['entrypoint.dart.lib.js'], + ); + + expect(output, isNotNull); + expect(output.errorCount, 0); + + final jsOutputFile = File( + jsFESOutputPath.replaceFirst('.dart.lib.js', '.ddc.js'), + ); + expect(jsOutputFile.existsSync(), isTrue); + final content = jsOutputFile.readAsStringSync(); + expect(content, contains('function main()')); + }); + }); +} diff --git a/build_modules/test/module_builder_test.dart b/build_modules/test/module_builder_test.dart index 01a065e9b8..3a684b5cfe 100644 --- a/build_modules/test/module_builder_test.dart +++ b/build_modules/test/module_builder_test.dart @@ -48,8 +48,14 @@ void main() { 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( metaModule.toJson(), ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), 'a|lib/c$moduleLibraryExtension': ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), 'a|lib/e$moduleLibraryExtension': ModuleLibrary.fromSource(assetE, '').serialize(), }, @@ -77,7 +83,6 @@ void main() { true, ); final moduleC = Module(assetC, [assetC], [], platform, true); - final moduleD = Module( assetD, [assetD, assetE], @@ -86,7 +91,13 @@ void main() { false, ); final moduleE = Module(assetE, [assetE], [], platform, true); - final metaModule = MetaModule([moduleA, moduleB, moduleD]); + final metaModule = MetaModule([ + moduleA, + moduleB, + moduleC, + moduleD, + moduleE, + ]); await testBuilder( ModuleBuilder(platform, usesWebHotReload: true), { @@ -95,11 +106,17 @@ void main() { 'a|lib/c.dart': '', 'a|lib/d.dart': '', 'a|lib/e.dart': '', - 'a|lib/${metaModuleCleanExtension(platform)}': jsonEncode( + 'a|lib/${metaModuleExtension(platform)}': jsonEncode( metaModule.toJson(), ), + 'a|lib/a$moduleLibraryExtension': + ModuleLibrary.fromSource(assetA, '').serialize(), + 'a|lib/b$moduleLibraryExtension': + ModuleLibrary.fromSource(assetB, '').serialize(), 'a|lib/c$moduleLibraryExtension': ModuleLibrary.fromSource(assetC, '').serialize(), + 'a|lib/d$moduleLibraryExtension': + ModuleLibrary.fromSource(assetD, '').serialize(), 'a|lib/e$moduleLibraryExtension': ModuleLibrary.fromSource(assetE, '').serialize(), }, From 12a8af7291dc8b4ccae3f14597ac61da52bb137d Mon Sep 17 00:00:00 2001 From: MarkZ Date: Wed, 1 Oct 2025 15:29:55 -0700 Subject: [PATCH 11/19] Adding reset to frontend server driver --- build_modules/lib/src/frontend_server_driver.dart | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index 00c9edc303..5a91444602 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -129,6 +129,14 @@ class FrontendServerProxyDriver { } } + /// Clears the proxy driver's state between invocations of separate apps. + void reset() { + _requestQueue.clear(); + _isProcessing = false; + _cachedOutput = null; + _frontendServer?.reset(); + } + Future terminate() async { await _frontendServer!.shutdown(); _frontendServer = null; @@ -243,6 +251,11 @@ class PersistentFrontendServer { return _stdoutHandler.compilerOutput!.future; } + void reset() { + _stdoutHandler.reset(); + _stdinController.add('reset'); + } + /// Records all modified files into the in-memory filesystem. void recordFiles() { final outputDillPath = outputDillUri.toFilePath(); From b9e30b39225c3bd17350e6cc001d8a5942fcc1f0 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Wed, 1 Oct 2025 15:30:27 -0700 Subject: [PATCH 12/19] Adding a builder for labeling the app entrypoint. --- build_modules/lib/src/module_builder.dart | 7 --- build_web_compilers/build.yaml | 14 ++++++ build_web_compilers/lib/builders.dart | 4 ++ .../src/web_entrypoint_marker_builder.dart | 46 +++++++++++++++++++ 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 build_web_compilers/lib/src/web_entrypoint_marker_builder.dart diff --git a/build_modules/lib/src/module_builder.dart b/build_modules/lib/src/module_builder.dart index 0e149d9568..f421ff3136 100644 --- a/build_modules/lib/src/module_builder.dart +++ b/build_modules/lib/src/module_builder.dart @@ -63,13 +63,6 @@ class ModuleBuilder implements Builder { serializedLibrary, ); final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); - if (usesWebHotReload && - libraryModule.isEntryPoint && - libraryModule.hasMain) { - // We must save the main entrypoint as the recompilation target for the - // Frontend Server before any JS files are emitted. - scratchSpace.entrypointAssetId = libraryModule.id; - } if (outputModule == null && libraryModule.hasMain) { outputModule = metaModule.modules.firstWhere( (m) => m.sources.contains(buildStep.inputId), diff --git a/build_web_compilers/build.yaml b/build_web_compilers/build.yaml index e8c23af08b..841a158034 100644 --- a/build_web_compilers/build.yaml +++ b/build_web_compilers/build.yaml @@ -1,6 +1,8 @@ targets: $default: builders: + build_web_compilers:entrypoint_marker: + enabled: true build_web_compilers:entrypoint: options: compiler: dart2js @@ -146,6 +148,18 @@ builders: compiler: dart2js applies_builders: - build_web_compilers:dart2js_archive_extractor + entrypoint_marker: + import: "package:build_web_compilers/builders.dart" + builder_factories: + - webEntrypointMarkerBuilder + build_extensions: + $web$: + - .web.entrypoint.json + required_inputs: + - .dart + build_to: cache + auto_apply: root_package + runs_before: ["build_web_compilers:ddc"] _stack_trace_mapper_copy: import: "tool/copy_builder.dart" builder_factories: diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index ad674dd8c9..a5a22aa5e7 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -11,6 +11,7 @@ import 'src/common.dart'; import 'src/ddc_frontend_server_builder.dart'; import 'src/sdk_js_compile_builder.dart'; import 'src/sdk_js_copy_builder.dart'; +import 'src/web_entrypoint_marker_builder.dart'; // Shared entrypoint builder Builder webEntrypointBuilder(BuilderOptions options) { @@ -18,6 +19,9 @@ Builder webEntrypointBuilder(BuilderOptions options) { return WebEntrypointBuilder.fromOptions(options); } +Builder webEntrypointMarkerBuilder(BuilderOptions options) => + WebEntrypointMarkerBuilder(); + // DDC related builders Builder ddcMetaModuleBuilder(BuilderOptions options) { _ensureSameDdcHotReloadOptions(options); diff --git a/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart b/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart new file mode 100644 index 0000000000..c379a78875 --- /dev/null +++ b/build_web_compilers/lib/src/web_entrypoint_marker_builder.dart @@ -0,0 +1,46 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; +import 'package:glob/glob.dart'; + +/// A builder that gathers information about a web target's 'main' entrypoint. +class WebEntrypointMarkerBuilder implements Builder { + @override + final buildExtensions = const { + r'$web$': ['.web.entrypoint.json'], + }; + + @override + Future build(BuildStep buildStep) async { + final scratchSpace = await buildStep.fetchResource(scratchSpaceResource); + + final webAssets = await buildStep.findAssets(Glob('web/**')).toList(); + final webEntrypointJson = {}; + + for (final asset in webAssets) { + if (asset.extension == '.dart') { + final moduleLibrary = ModuleLibrary.fromSource( + asset, + await buildStep.readAsString(asset), + ); + if (moduleLibrary.hasMain && moduleLibrary.isEntryPoint) { + // We must save the main entrypoint as the recompilation target for + // the Frontend Server before any JS files are emitted. + scratchSpace.entrypointAssetId = asset; + webEntrypointJson['entrypoint'] = asset.uri.toString(); + break; + } + } + } + + await buildStep.writeAsString( + AssetId(buildStep.inputId.package, 'web/.web.entrypoint.json'), + jsonEncode(webEntrypointJson), + ); + } +} From 309b11f6ce324a0b92a34a9f702eaa22462cd2f5 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Wed, 1 Oct 2025 15:32:57 -0700 Subject: [PATCH 13/19] Removing the entrypoint from builder options --- build_web_compilers/lib/builders.dart | 15 +-------------- .../lib/src/ddc_frontend_server_builder.dart | 4 +--- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/build_web_compilers/lib/builders.dart b/build_web_compilers/lib/builders.dart index a5a22aa5e7..ea74aeec0c 100644 --- a/build_web_compilers/lib/builders.dart +++ b/build_web_compilers/lib/builders.dart @@ -47,14 +47,7 @@ Builder ddcBuilder(BuilderOptions options) { _ensureSameDdcOptions(options); if (_readWebHotReloadOption(options)) { - final entrypoint = _readEntrypoint(options); - if (entrypoint == null) { - throw StateError( - "DDC's Frontend Server configuration requires the " - '`entrypoint` to be specified in the `build.yaml`.', - ); - } - return DdcFrontendServerBuilder(entrypoint: entrypoint); + return DdcFrontendServerBuilder(); } return DevCompilerBuilder( @@ -190,10 +183,6 @@ bool _readTrackInputsCompilerOption(BuilderOptions options) { return options.config[_trackUnusedInputsCompilerOption] as bool? ?? true; } -String? _readEntrypoint(BuilderOptions options) { - return options.config[_entrypoint] as String?; -} - bool _readWebHotReloadOption(BuilderOptions options) { return options.config[_webHotReloadOption] as bool? ?? false; } @@ -211,7 +200,6 @@ const _emitDebugSymbolsOption = 'emit-debug-symbols'; const _canaryOption = 'canary'; const _trackUnusedInputsCompilerOption = 'track-unused-inputs'; const _environmentOption = 'environment'; -const _entrypoint = 'entrypoint'; const _webHotReloadOption = 'web-hot-reload'; const _supportedOptions = [ @@ -221,6 +209,5 @@ const _supportedOptions = [ _emitDebugSymbolsOption, _canaryOption, _trackUnusedInputsCompilerOption, - _entrypoint, _webHotReloadOption, ]; diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index ca8be764a1..883e431d51 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -14,9 +14,7 @@ import 'platforms.dart'; /// A builder that compiles DDC modules with the Frontend Server. class DdcFrontendServerBuilder implements Builder { - final String entrypoint; - - DdcFrontendServerBuilder({required this.entrypoint}); + DdcFrontendServerBuilder(); @override final Map> buildExtensions = { From 625a922f11764e9f0ff42955cfe170b1c940754b Mon Sep 17 00:00:00 2001 From: MarkZ Date: Wed, 1 Oct 2025 15:59:08 -0700 Subject: [PATCH 14/19] Fixing error handling --- build_web_compilers/lib/src/ddc_frontend_server_builder.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 883e431d51..0236b7aa5b 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -50,9 +50,7 @@ class DdcFrontendServerBuilder implements Builder { try { await _compile(module, buildStep); - } on DartDevcCompilationException catch (e) { - await handleError(e); - } on MissingModulesException catch (e) { + } catch (e) { await handleError(e); } } From 150a42ae9d3a9561df0dd71786c5c7bcafb1c64f Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 3 Oct 2025 00:28:32 -0700 Subject: [PATCH 15/19] Fixing some shutdown race conditions --- build_modules/lib/src/frontend_server_driver.dart | 14 +++++++------- build_modules/lib/src/workers.dart | 2 +- .../lib/src/ddc_frontend_server_builder.dart | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/build_modules/lib/src/frontend_server_driver.dart b/build_modules/lib/src/frontend_server_driver.dart index 5a91444602..8d6a0a4116 100644 --- a/build_modules/lib/src/frontend_server_driver.dart +++ b/build_modules/lib/src/frontend_server_driver.dart @@ -220,17 +220,17 @@ class PersistentFrontendServer { ); } - Future compile(String entrypoint) { + Future compile(String entrypoint) async { _stdoutHandler.reset(); _stdinController.add('compile $entrypoint'); - return _stdoutHandler.compilerOutput!.future; + return await _stdoutHandler.compilerOutput!.future; } /// Either [accept] or [reject] should be called after every [recompile] call. Future recompile( String entrypoint, List invalidatedFiles, - ) { + ) async { _stdoutHandler.reset(); final inputKey = const Uuid().v4(); _stdinController.add('recompile $entrypoint $inputKey'); @@ -238,17 +238,17 @@ class PersistentFrontendServer { _stdinController.add(file.toString()); } _stdinController.add(inputKey); - return _stdoutHandler.compilerOutput!.future; + return await _stdoutHandler.compilerOutput!.future; } void accept() { _stdinController.add('accept'); } - Future reject() { + Future reject() async { _stdoutHandler.reset(expectSources: false); _stdinController.add('reject'); - return _stdoutHandler.compilerOutput!.future; + return await _stdoutHandler.compilerOutput!.future; } void reset() { @@ -279,8 +279,8 @@ class PersistentFrontendServer { Future shutdown() async { _stdinController.add('quit'); - await _stdinController.close(); await _server?.exitCode; + await _stdinController.close(); _server = null; } } diff --git a/build_modules/lib/src/workers.dart b/build_modules/lib/src/workers.dart index 55b70117af..6278c13391 100644 --- a/build_modules/lib/src/workers.dart +++ b/build_modules/lib/src/workers.dart @@ -126,8 +126,8 @@ FrontendServerProxyDriver? __frontendServerProxyDriver; final frontendServerProxyDriverResource = Resource( () async => _frontendServerProxyDriver, beforeExit: () async { - await __frontendServerProxyDriver?.terminate(); _frontendServerProxyWorkersAreDoneCompleter?.complete(); + await __frontendServerProxyDriver?.terminate(); _frontendServerProxyWorkersAreDoneCompleter = null; __frontendServerProxyDriver = null; }, diff --git a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart index 0236b7aa5b..6909d781a4 100644 --- a/build_web_compilers/lib/src/ddc_frontend_server_builder.dart +++ b/build_web_compilers/lib/src/ddc_frontend_server_builder.dart @@ -9,7 +9,6 @@ import 'package:build/build.dart'; import 'package:build_modules/build_modules.dart'; import 'common.dart'; -import 'errors.dart'; import 'platforms.dart'; /// A builder that compiles DDC modules with the Frontend Server. From c48e0152519bcbd4fc31aa56fbfdcb9cab28616b Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 3 Oct 2025 00:29:02 -0700 Subject: [PATCH 16/19] Adding reader-writer resets --- build_test/lib/src/internal_test_reader_writer.dart | 6 ++++++ build_test/lib/src/test_reader_writer.dart | 3 +++ 2 files changed, 9 insertions(+) diff --git a/build_test/lib/src/internal_test_reader_writer.dart b/build_test/lib/src/internal_test_reader_writer.dart index a1fb1505e3..87de6a8ed8 100644 --- a/build_test/lib/src/internal_test_reader_writer.dart +++ b/build_test/lib/src/internal_test_reader_writer.dart @@ -274,4 +274,10 @@ class _ReaderWriterTestingImpl implements ReaderWriterTesting { void delete(AssetId id) => _readerWriter.filesystem.deleteSync( _readerWriter.assetPathProvider.pathFor(id), ); + + @override + void reset() { + _readerWriter.assetsRead.clear(); + _readerWriter.assetsWritten.clear(); + } } diff --git a/build_test/lib/src/test_reader_writer.dart b/build_test/lib/src/test_reader_writer.dart index a9ddd000db..cdfe164564 100644 --- a/build_test/lib/src/test_reader_writer.dart +++ b/build_test/lib/src/test_reader_writer.dart @@ -92,4 +92,7 @@ abstract interface class ReaderWriterTesting { /// Deletes [id] from the [TestReaderWriter] in-memory filesystem. void delete(AssetId id); + + /// Resets state in this [TestReaderWriter] between rebuilds. + void reset(); } From 9349807fd381007a10d2caed35c8c76d8abd1b38 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 3 Oct 2025 00:34:41 -0700 Subject: [PATCH 17/19] Adding support for incremental build testing. --- build_test/lib/src/test_builder.dart | 49 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/build_test/lib/src/test_builder.dart b/build_test/lib/src/test_builder.dart index e121086c0d..f6978d8a05 100644 --- a/build_test/lib/src/test_builder.dart +++ b/build_test/lib/src/test_builder.dart @@ -9,6 +9,8 @@ import 'package:build/build.dart'; import 'package:build/experiments.dart'; import 'package:build_config/build_config.dart'; // ignore: implementation_imports +import 'package:build_runner/src/build/asset_graph/graph.dart'; +// ignore: implementation_imports import 'package:build_runner/src/internal.dart'; import 'package:built_collection/built_collection.dart'; import 'package:glob/glob.dart'; @@ -137,7 +139,9 @@ Future testBuilder( PackageConfig? packageConfig, Resolvers? resolvers, TestReaderWriter? readerWriter, + AssetGraph? assetGraph, bool enableLowResourceMode = false, + bool performCleanup = true, }) async { return testBuilders( [builder], @@ -151,7 +155,9 @@ Future testBuilder( packageConfig: packageConfig, resolvers: resolvers, readerWriter: readerWriter, + assetGraph: assetGraph, enableLowResourceMode: enableLowResourceMode, + performCleanup: performCleanup, ); } @@ -182,7 +188,9 @@ Future testBuilders( Map> appliesBuilders = const {}, bool testingBuilderConfig = true, TestReaderWriter? readerWriter, + AssetGraph? assetGraph, bool enableLowResourceMode = false, + bool performCleanup = true, }) { final builderFactories = []; final optionalBuilderFactories = Set.identity(); @@ -223,6 +231,7 @@ Future testBuilders( testingBuilderConfig: testingBuilderConfig, readerWriter: readerWriter, enableLowResourceMode: enableLowResourceMode, + performCleanup: performCleanup, ); } @@ -306,7 +315,9 @@ Future testBuilderFactories( Map> appliesBuilders = const {}, bool testingBuilderConfig = true, TestReaderWriter? readerWriter, + AssetGraph? assetGraph, bool enableLowResourceMode = false, + bool performCleanup = true, }) async { onLog ??= _printOnFailureOrWrite; @@ -336,16 +347,18 @@ Future testBuilderFactories( } rootPackage ??= allPackages.first; - readerWriter ??= TestReaderWriter(rootPackage: rootPackage); - - sourceAssets.forEach((serializedId, contents) { - final id = makeAssetId(serializedId); - if (contents is String) { - readerWriter!.testing.writeString(id, contents); - } else if (contents is List) { - readerWriter!.testing.writeBytes(id, contents); - } - }); + // Only write source assets if this is a new build (non-incremental). + if (readerWriter == null) { + readerWriter = TestReaderWriter(rootPackage: rootPackage); + sourceAssets.forEach((serializedId, contents) { + final id = makeAssetId(serializedId); + if (contents is String) { + readerWriter!.testing.writeString(id, contents); + } else if (contents is List) { + readerWriter!.testing.writeBytes(id, contents); + } + }); + } final inputFilter = isInput ?? generateFor?.contains ?? (_) => true; inputIds.retainWhere((id) => inputFilter('$id')); @@ -464,13 +477,17 @@ Future testBuilderFactories( ); await buildPlan.deleteFilesAndFolders(); + final previousAssetGraph = assetGraph ?? buildPlan.takePreviousAssetGraph(); + final buildSeries = BuildSeries(buildPlan); // Run the build. final buildResult = await buildSeries.run({}, recentlyBootstrapped: true); - // Do cleanup that would usually happen on process exit. - await buildSeries.close(); + if (performCleanup) { + // Do cleanup that would usually happen on process exit. + await buildSeries.close(); + } // Stop logging. buildLog.configuration = buildLog.configuration.rebuild((b) { @@ -482,6 +499,7 @@ Future testBuilderFactories( return TestBuilderResult( buildResult: buildResult, + assetGraph: previousAssetGraph, readerWriter: readerWriter, ); } @@ -489,8 +507,13 @@ Future testBuilderFactories( class TestBuilderResult { final BuildResult buildResult; final TestReaderWriter readerWriter; + final AssetGraph? assetGraph; - TestBuilderResult({required this.buildResult, required this.readerWriter}); + TestBuilderResult({ + required this.buildResult, + required this.readerWriter, + this.assetGraph, + }); } void _printOnFailureOrWrite(LogRecord record) { From f108cb6663375a65f33aa6aa27d9c0bb69ae910e Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 3 Oct 2025 00:35:06 -0700 Subject: [PATCH 18/19] Adding DDC frontend server builder tests --- .../ddc_frontend_server_builder_test.dart | 299 ++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 build_web_compilers/test/ddc_frontend_server_builder_test.dart diff --git a/build_web_compilers/test/ddc_frontend_server_builder_test.dart b/build_web_compilers/test/ddc_frontend_server_builder_test.dart new file mode 100644 index 0000000000..1de9fbbf8d --- /dev/null +++ b/build_web_compilers/test/ddc_frontend_server_builder_test.dart @@ -0,0 +1,299 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@Tags(['integration']) +library; + +import 'package:build/build.dart'; +import 'package:build_modules/build_modules.dart'; +import 'package:build_runner/src/build/asset_graph/graph.dart'; +import 'package:build_test/build_test.dart'; +import 'package:build_web_compilers/build_web_compilers.dart'; +import 'package:build_web_compilers/builders.dart'; +import 'package:build_web_compilers/src/ddc_frontend_server_builder.dart'; +import 'package:logging/logging.dart'; +import 'package:test/test.dart'; + +void main() { + group('DdcFrontendServerBuilder', () { + final assets = { + 'a|web/main.dart': ''' + import 'package:b/b.dart'; + void main() { + print(helloWorld); + } + ''', + 'b|lib/b.dart': "const helloWorld = 'Hello World!';", + }; + + final builders = [ + const ModuleLibraryBuilder(), + ddcMetaModuleBuilder(const BuilderOptions({'web-hot-reload': true})), + ddcModuleBuilder(const BuilderOptions({'web-hot-reload': true})), + DdcFrontendServerBuilder(), + ]; + + final allOutputs = { + 'a|web/main.module.library': isNotEmpty, + 'a|lib/.ddc.meta_module.raw': isNotEmpty, + 'a|web/main.ddc.module': isNotEmpty, + 'a|web/main.ddc.js': decodedMatches(contains('function main')), + 'a|web/main.ddc.js.map': isNotEmpty, + 'a|web/main.ddc.js.metadata': isNotEmpty, + 'b|lib/b.module.library': isNotEmpty, + 'b|lib/.ddc.meta_module.raw': isNotEmpty, + 'b|lib/b.ddc.module': isNotEmpty, + 'b|lib/b.ddc.js': decodedMatches(contains('Hello World!')), + 'b|lib/b.ddc.js.map': isNotEmpty, + 'b|lib/b.ddc.js.metadata': isNotEmpty, + }; + + // ignore: unused_local_variable + final nonJsOutputs = { + 'a|web/main.module.library': isNotEmpty, + 'a|lib/.ddc.meta_module.raw': isNotEmpty, + 'a|web/main.ddc.module': isNotEmpty, + 'b|lib/b.module.library': isNotEmpty, + 'b|lib/.ddc.meta_module.raw': isNotEmpty, + 'b|lib/b.ddc.module': isNotEmpty, + }; + + final aJsOutputs = { + 'a|web/main.ddc.js': decodedMatches(contains('function main')), + 'a|web/main.ddc.js.map': isNotEmpty, + 'a|web/main.ddc.js.metadata': isNotEmpty, + }; + + // ignore: unused_local_variable + final aModuleOutputs = { + 'a|web/main.module.library': isNotEmpty, + 'a|web/main.ddc.module': isNotEmpty, + }; + + final bJsOutputs = { + 'b|lib/b.ddc.js': decodedMatches(contains('Hello World!')), + 'b|lib/b.ddc.js.map': isNotEmpty, + 'b|lib/b.ddc.js.metadata': isNotEmpty, + }; + + final bModuleOutputs = { + 'b|lib/b.module.library': isNotEmpty, + 'b|lib/b.ddc.module': isNotEmpty, + }; + + setUp(() async { + final listener = Logger.root.onRecord.listen( + (r) => printOnFailure('$r\n${r.error}\n${r.stackTrace}'), + ); + addTearDown(listener.cancel); + }); + + /// [testBuilders] doesn't respect runs_before in build.yaml, so we must run + /// this before every invocation to ensure that side effects appear in the + /// right order. + Future runWebEntrypointMarker( + String rootPackage, { + TestReaderWriter? readerWriter, + AssetGraph? assetGraph, + + void Function(LogRecord log)? onLog, + }) async { + await testBuilder( + webEntrypointMarkerBuilder(const BuilderOptions({})), + assets, + outputs: {'$rootPackage|web/.web.entrypoint.json': isNotEmpty}, + rootPackage: rootPackage, + readerWriter: readerWriter, + assetGraph: assetGraph, + onLog: onLog, + performCleanup: false, + ); + } + + test('can compile a simple app', () async { + final logs = []; + await runWebEntrypointMarker('a', onLog: logs.add); + await testBuilders( + builders, + assets, + outputs: allOutputs, + // onLog: logs.add, + rootPackage: 'a', + performCleanup: true, + ); + + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + }); + + test('can recompile incrementally after valid edits', () async { + final logs = []; + await runWebEntrypointMarker('a'); + var result = await testBuilders( + builders, + assets, + outputs: allOutputs, + onLog: logs.add, + performCleanup: false, + rootPackage: 'a', + ); + + // Make a simple edit and re-run the build. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), ''' + const helloWorld = 'Hello Dash!'; + '''); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...aJsOutputs, + ...bJsOutputs, + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Dash!')), + 'b|lib/b.ddc.js': decodedMatches(contains('Hello Dash!')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: true, + ); + + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + }); + + test('can recompile incrementally after invalid edits', () async { + final logs = []; + await runWebEntrypointMarker('a'); + var result = await testBuilders( + builders, + assets, + outputs: { + ...allOutputs, + 'a|web/main.ddc.js': decodedMatches(contains('function main')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: false, + ); + + // Introduce a generic class and re-run the build. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), ''' + class Foo{} + const helloWorld = 'Hello Dash!'; + '''); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...aJsOutputs, + ...bJsOutputs, + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Dash!')), + 'b|lib/b.ddc.js': decodedMatches(contains('Hello Dash!')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: false, + ); + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + logs.clear(); + + // Change the number of generic parameters, which is invalid for hot + // reload. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), """ + class Foo{} + const helloWorld = 'Hello Python!'; + """); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Python!')), + 'a|web/main$jsModuleErrorsExtension': decodedMatches( + allOf( + contains('Exception'), + contains('Frontend Server encountered errors during compilation'), + contains('Limitation'), + ), + ), + 'b|lib/b$jsModuleErrorsExtension': decodedMatches( + allOf( + contains('Exception'), + contains('Frontend Server encountered errors during compilation'), + contains('Limitation'), + ), + ), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: false, + ); + + expect( + logs, + contains( + predicate( + (record) => + record.level == Level.SEVERE && + record.message.contains('Exception') && + record.message.contains( + 'Frontend Server encountered errors during compilation', + ) && + record.message.contains('Limitation'), + ), + ), + ); + logs.clear(); + + // Revert the number of generic parameters and successfully rebuild. + result.readerWriter.testing.reset(); + await result.readerWriter.writeAsString(AssetId.parse('b|lib/b.dart'), """ + class Foo{} + const helloWorld = 'Hello Golang!'; + """); + await runWebEntrypointMarker('a'); + result = await testBuilders( + builders, + assets, + readerWriter: result.readerWriter, + assetGraph: result.assetGraph, + outputs: { + ...aJsOutputs, + ...bJsOutputs, + ...bModuleOutputs, + 'b|lib/b.dart': decodedMatches(contains('Hello Golang!')), + 'b|lib/b.ddc.js': decodedMatches(contains('Hello Golang!')), + }, + onLog: logs.add, + rootPackage: 'a', + performCleanup: true, + ); + + expect( + logs, + isNot(anyOf(logs.map((r) => r.level >= Level.WARNING))), + ); + }); + }); +} From 75f3061d2a7913a9e6946f70e9559a4798e47dd1 Mon Sep 17 00:00:00 2001 From: MarkZ Date: Fri, 3 Oct 2025 00:48:34 -0700 Subject: [PATCH 19/19] Updating entrypoint marker builder --- build_web_compilers/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_web_compilers/build.yaml b/build_web_compilers/build.yaml index 841a158034..52ed081832 100644 --- a/build_web_compilers/build.yaml +++ b/build_web_compilers/build.yaml @@ -159,7 +159,9 @@ builders: - .dart build_to: cache auto_apply: root_package - runs_before: ["build_web_compilers:ddc"] + applies_builders: + - build_web_compilers:entrypoint + runs_before: ["build_web_compilers:ddc", "build_web_compilers:ddc_modules"] _stack_trace_mapper_copy: import: "tool/copy_builder.dart" builder_factories: