chore(cli): Re-implement celest build (#279)

- Reimplement `celest build` to use containers maintained by others instead of Celest-specific containers
- Add container tests

Author: dnys1

.gitignore

@@ -4,6 +4,8 @@ doc/api/
 **/doc/api/
 pubspec_overrides.yaml
 pubspec.lock
+.flutter-plugins
+.flutter-plugins-dependencies
 
 ## RUST ##
 # will have compiled files and executables

apps/cli/fixtures/fixtures_test.dart

@@ -3,13 +3,15 @@
 @Tags(['goldens'])
 library;
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io' hide Directory;
 import 'dart:isolate';
 import 'dart:math';
 
 import 'package:async/async.dart';
 import 'package:celest/src/runtime/serve.dart';
+import 'package:celest_ast/celest_ast.dart' as ast;
 import 'package:celest_cli/src/analyzer/analysis_result.dart';
 import 'package:celest_cli/src/analyzer/celest_analyzer.dart';
 import 'package:celest_cli/src/codegen/client_code_generator.dart';
@@ -18,12 +20,15 @@ import 'package:celest_cli/src/compiler/api/local_api_runner.dart';
 import 'package:celest_cli/src/context.dart';
 import 'package:celest_cli/src/database/project/project_database.dart';
 import 'package:celest_cli/src/env/config_value_solver.dart';
+import 'package:celest_cli/src/frontend/celest_frontend.dart';
 import 'package:celest_cli/src/init/project_migrator.dart';
+import 'package:celest_cli/src/process/port_finder.dart';
 import 'package:celest_cli/src/project/celest_project.dart';
 import 'package:celest_cli/src/project/project_linker.dart';
 import 'package:celest_cli/src/pub/pub_action.dart';
 import 'package:celest_cli/src/sdk/dart_sdk.dart';
 import 'package:celest_cli/src/sdk/sdk_finder.dart';
+import 'package:celest_cli/src/utils/cli.dart';
 import 'package:celest_cli/src/utils/recase.dart';
 import 'package:dart_jsonwebtoken/dart_jsonwebtoken.dart';
 import 'package:file/file.dart';
@@ -189,6 +194,7 @@ class TestRunner {
       testResolve();
       testCodegen();
       testClient();
+      testBuild();
 
       final apisDir = fileSystem.directory(
         p.join(projectRoot, 'lib', 'src', 'functions'),
@@ -372,6 +378,78 @@ class TestRunner {
     });
   }
 
+  void testBuild() {
+    // Docker only available on Linux runner in CI.
+    final isCI = platform.environment.containsKey('CI');
+    final shouldRun = !isCI || platform.isLinux;
+    test('build', skip: !shouldRun, () async {
+      final CelestAnalysisResult(:project, :errors) =
+          await analyzer.analyzeProject(
+        migrateProject: false,
+        updateResources: updateGoldens,
+      );
+      expect(errors, isEmpty);
+      expect(project, isNotNull);
+
+      final frontend = CelestFrontend();
+      final result = await frontend.build(
+        migrateProject: false,
+        currentProgress: cliLogger.progress('Building project...'),
+        environmentId: 'production',
+      );
+      expect(result, 0);
+
+      final outputDir = projectPaths.buildDir;
+      final imageName = '$testName-${Random().nextInt(1000000)}';
+      final dockerBuild = await processManager.run(
+        ['docker', 'build', '-t', imageName, '.'],
+        workingDirectory: outputDir,
+      );
+      expect(
+        dockerBuild.exitCode,
+        0,
+        reason: '${dockerBuild.stdout}\n${dockerBuild.stderr}',
+      );
+
+      addTearDown(() async {
+        await processManager.run(
+          ['docker', 'image', 'rm', imageName],
+        );
+      });
+
+      final openPort = await RandomPortFinder().findOpenPort();
+      final dockerRun = await processManager.start(
+        [
+          'docker',
+          'run',
+          '--rm',
+          '-p',
+          '$openPort:8080',
+          for (final database in project!.databases.values)
+            if (database.config case ast.CelestDatabaseConfig(:final hostname))
+              '--env=${hostname.name}=file::memory:',
+          if (project.auth?.providers.isNotEmpty ?? false)
+            '--env=CELEST_AUTH_DATABASE_HOST=file::memory:',
+          imageName,
+        ],
+      );
+      addTearDown(() => dockerRun.kill(ProcessSignal.sigkill));
+
+      final dockerOutput = StreamController<String>.broadcast(sync: true);
+      unawaited(dockerRun.exitCode.then((_) => dockerOutput.close()));
+
+      dockerRun
+        ..captureStdout()
+        ..captureStdout(sink: dockerOutput.add)
+        ..captureStderr()
+        ..captureStderr(sink: dockerOutput.add);
+      await expectLater(
+        dockerOutput.stream,
+        emitsThrough('Serving on http://localhost:8080'),
+      );
+    });
+  }
+
   void testApis(Directory apisDir, List<String>? includeApis) {
     final apis = testCases?.apis ?? const {};
     if (apis.isEmpty) {
@@ -417,7 +495,15 @@ class TestRunner {
         apiRunner = await LocalApiRunner.start(
           resolvedProject: projectResolver.resolvedProject,
           path: entrypoint,
-          configValues: configValues,
+          configValues: {
+            ...configValues,
+            for (final database in project.databases.values)
+              if (database.config
+                  case ast.CelestDatabaseConfig(:final hostname))
+                hostname.name: 'file::memory:',
+            if (project.auth?.providers.isNotEmpty ?? false)
+              'CELEST_AUTH_DATABASE_HOST': 'file::memory:',
+          },
           environmentId: 'local',
           verbose: false,
           stdoutPipe: logSink,

apps/cli/fixtures/standalone/data/goldens/ast.json

@@ -1,59 +1,101 @@
 import 'package:celest_ast/celest_ast.dart' as ast;
 import 'package:celest_ast/celest_ast.dart';
 import 'package:celest_cli/src/utils/error.dart';
+import 'package:mustache_template/mustache_template.dart';
 
 /// Generates a `Dockerfile` for the user's project so they can self-host it.
 final class DockerfileGenerator {
   DockerfileGenerator({required this.project});
 
   final ast.Project project;
 
-  static const String _dartTemplate = r'''
-FROM celestdev/dart-builder:{{version}} AS build
+  static final Template _dartTemplate = Template(r'''
+# syntax=docker/dockerfile:1
+FROM dart:{{version}} AS build
 
 WORKDIR /app
+{{ #includes_data }}
+# Add SQLite3
+RUN apt update && apt install -y libsqlite3-0
+RUN cp $(find / -name libsqlite3.so* -type f | head -n1) /app/libsqlite3.so
+{{ /includes_data }}
 COPY celest.aot.dill main.aot.dill
 
 RUN [ "/usr/lib/dart/bin/utils/gen_snapshot", "--snapshot_kind=app-aot-elf", "--elf=/app/main.aot", "/app/main.aot.dill" ]
 
-FROM celestdev/dart-runtime:{{version}}
+FROM scratch
+
+COPY --from=build /runtime /
 
 WORKDIR /app
+COPY --from=build /usr/lib/dart/bin/dartaotruntime .
+COPY --from=build /app ./
 COPY celest.json .
-COPY --from=build /app/main.aot .
 
 ENV PORT=8080
 EXPOSE 8080
-''';
 
-  static const String _flutterTemplate = r'''
-FROM celestdev/flutter-builder:{{version}} AS build
+ENTRYPOINT [ "/app/dartaotruntime" ]
+CMD [ "/app/main.aot" ]
+''');
 
-WORKDIR /app
-COPY celest.aot.dill main.aot.dill
+  static final Template _flutterTemplate = Template(r'''
+# syntax=docker/dockerfile:1
+ARG DEBIAN_VERSION=12
 
-RUN [ "/usr/lib/dart/bin/utils/gen_snapshot", "--snapshot_kind=app-aot-elf", "--elf=/app/main.aot", "/app/main.aot.dill" ]
+FROM debian:${DEBIAN_VERSION}-slim
+
+ARG TARGETARCH
 
-FROM celestdev/flutter-runtime:{{version}}
+# Set up fonts for the Flutter engine
+RUN apt update && apt install -y \
+    fontconfig \
+    fonts-cantarell \
+    fonts-liberation2
+RUN fc-cache -f
+
+# Add CA certificates
+RUN apt install -y ca-certificates
+
+WORKDIR /celest
+{{ #includes_data }}
+# Add SQLite3
+RUN apt install -y libsqlite3-0
+RUN cp $(find / -name libsqlite3.so* -type f | head -n1) /celest/libsqlite3.so
+{{ /includes_data }}
+COPY --from=ghcr.io/cirruslabs/flutter:{{version}} /sdks/flutter/bin/cache/artifacts/engine/linux-${TARGETARCH/amd64/x64}/icudtl.dat .
+COPY --from=ghcr.io/cirruslabs/flutter:{{version}} /sdks/flutter/bin/cache/artifacts/engine/linux-${TARGETARCH/amd64/x64}/*.so ./
+COPY --from=ghcr.io/cirruslabs/flutter:{{version}} /sdks/flutter/bin/cache/artifacts/engine/linux-${TARGETARCH/amd64/x64}/*.so* ./
+COPY --from=ghcr.io/cirruslabs/flutter:{{version}} /sdks/flutter/bin/cache/artifacts/engine/linux-${TARGETARCH/amd64/x64}/flutter_tester flutter_runner
+
+# Clean up
+RUN apt-get clean
 
 WORKDIR /app
+COPY flutter_assets/ ./
 COPY celest.json .
-COPY --from=build /app/main.aot .
 
+ENV LD_LIBRARY_PATH="/app:/celest:${LD_LIBRARY_PATH}"
 ENV PORT=8080
 EXPOSE 8080
-''';
+
+ENTRYPOINT [ "/celest/flutter_runner", "--non-interactive", "--run-forever", "--disable-vm-service", "--icu-data-file-path=/celest/icudtl.dat", "--verbose-logging", "--enable-platform-isolates", "--force-multithreading", "--cache-dir-path=/tmp", "--flutter-assets-dir=/app", "--snapshot-asset-path=/app" ]
+CMD [ "/app/kernel_blob.bin" ]
+''');
 
   String generate() {
+    // TODO(dnys1): Add this database to the project AST so that it's included
+    // in `project.databases`.
+    final hasAuthDatabase = project.auth?.providers.isNotEmpty ?? false;
     return switch (project.sdkConfig.targetSdk) {
-      SdkType.flutter => _flutterTemplate.replaceAll(
-          '{{version}}',
-          project.sdkConfig.flutter!.version.canonicalizedVersion,
-        ),
-      SdkType.dart => _dartTemplate.replaceAll(
-          '{{version}}',
-          project.sdkConfig.dart.version.canonicalizedVersion,
-        ),
+      SdkType.flutter => _flutterTemplate.renderString({
+          'version': project.sdkConfig.flutter!.version.canonicalizedVersion,
+          'includes_data': project.databases.isNotEmpty || hasAuthDatabase,
+        }),
+      SdkType.dart => _dartTemplate.renderString({
+          'version': project.sdkConfig.dart.version.canonicalizedVersion,
+          'includes_data': project.databases.isNotEmpty || hasAuthDatabase,
+        }),
       final unknown => unreachable('Unknown SDK: $unknown'),
     };
   }

apps/cli/fixtures/standalone/data/goldens/ast.resolved.json

@@ -29,9 +29,7 @@ import 'package:watcher/watcher.dart';
 enum RestartMode { hotReload, fullRestart }
 
 final class CelestFrontend {
-  factory CelestFrontend() => instance ??= CelestFrontend._();
-
-  CelestFrontend._() {
+  CelestFrontend() {
     // Initialize immediately instead of lazily since _stopSub is never accessed
     // directly until `close`.
     _stopSub = StreamGroup.merge([
@@ -491,13 +489,6 @@ final class CelestFrontend {
           resolvedProject: resolvedProject,
         );
         final outputs = codeGenerator.generate();
-        final outputsDir = Directory(projectPaths.outputsDir);
-        if (outputsDir.existsSync() && !_didFirstCompile) {
-          await outputsDir.delete(recursive: true);
-        }
-        if (stopped) {
-          throw const CancellationException('Celest was stopped');
-        }
         await (outputs.write(), celestProject.invalidate(outputs.keys)).wait;
         if (stopped) {
           throw const CancellationException('Celest was stopped');
@@ -530,24 +521,44 @@ final class CelestFrontend {
     required ResolvedProject resolvedProject,
     required String environmentId,
   }) async {
-    final entrypointCompiler = EntrypointCompiler(
-      logger: logger,
-      verbose: verbose,
-      enabledExperiments: celestProject.analysisOptions.enabledExperiments,
-    );
-    final kernel = await entrypointCompiler.compile(
-      resolvedProject: resolvedProject,
-      entrypointPath: projectPaths.localApiEntrypoint,
-    );
-
     final buildOutputs = fileSystem.directory(projectPaths.buildDir);
     if (!buildOutputs.existsSync()) {
       await buildOutputs.create(recursive: true);
     }
-
-    await buildOutputs
-        .childFile('celest.aot.dill')
-        .writeAsBytes(kernel.outputDill);
+    switch (resolvedProject.sdkConfig.targetSdk) {
+      case ast.SdkType.dart:
+        final entrypointCompiler = EntrypointCompiler(
+          logger: logger,
+          verbose: verbose,
+          enabledExperiments: celestProject.analysisOptions.enabledExperiments,
+        );
+        final kernel = await entrypointCompiler.compile(
+          resolvedProject: resolvedProject,
+          entrypointPath: projectPaths.localApiEntrypoint,
+        );
+        await buildOutputs
+            .childFile('celest.aot.dill')
+            .writeAsBytes(kernel.outputDill);
+      case ast.SdkType.flutter:
+        final bundleRes = await processManager.run(
+          [
+            'flutter',
+            'build',
+            'bundle',
+            '--packages=${projectPaths.packagesConfig}',
+            '--asset-dir=${p.join(buildOutputs.path, 'flutter_assets')}',
+            '--target=${projectPaths.localApiEntrypoint}',
+            '--target-platform=linux-x64',
+          ],
+          workingDirectory: projectPaths.projectRoot,
+        );
+        if (bundleRes.exitCode != 0) {
+          throw CliException(
+            'Failed to build project:\n'
+            '${bundleRes.stdout}\n${bundleRes.stderr}',
+          );
+        }
+    }
 
     final dockerfile = DockerfileGenerator(project: project);
     await buildOutputs

apps/cli/lib/src/codegen/api/dockerfile_generator.dart

@@ -51,6 +51,7 @@ dependencies:
   logging: ^1.2.0
   mason_logger: ^0.3.0
   meta: ^1.10.0
+  mustache_template: ^2.0.0
   native_assets_builder: ^0.8.1
   native_assets_cli: ^0.7.2
   native_storage: ^0.2.2
@@ -100,7 +101,6 @@ dev_dependencies:
   jose: ^0.3.4
   json_serializable: ^6.8.0
   mocktail: ^1.0.2
-  mustache_template: ^2.0.0
   test: ^1.24.9
   test_descriptor: ^2.0.1