Skip to content

Commit f2e4649

Browse files
dnys1dnys1
authored
feat(cli): Support cross-compilation (#350)
Dart 3.8 will support cross-compilation via the `dart compile exe` command. When available, use this instead for faster deployments.
1 parent 2a088c9 commit f2e4649

File tree

11 files changed

+281
-58
lines changed

11 files changed

+281
-58
lines changed

‎.github/workflows/distroless_core.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Distroless (Core)
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'apps/distroless/core/**'
8+
- '.github/workflows/distroless_core.yaml'
9+
10+
# Prevent duplicate runs due to Graphite
11+
# https://graphite.dev/docs/troubleshooting#why-are-my-actions-running-twice
12+
concurrency:
13+
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || ''}}
14+
cancel-in-progress: true
15+
16+
jobs:
17+
build_and_push:
18+
name: Build and Push Docker Image
19+
runs-on: ubuntu-large
20+
21+
steps:
22+
- name: Git Checkout
23+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
24+
25+
# Set up QEMU for multi-platform builds (needed for ARM on x86 runners)
26+
- name: Set up QEMU
27+
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # 3.6.0
28+
with:
29+
platforms: arm64
30+
31+
# Set up Docker Buildx, the builder engine for multi-platform builds
32+
- name: Set up Docker Buildx
33+
id: buildx
34+
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # 3.10.0
35+
# Optional: Specify a specific builder instance name
36+
# with:
37+
# driver-opts: image=moby/buildkit:v0.12.4 # Pin buildkit version if needed
38+
39+
# Login to Docker Hub using secrets
40+
- name: Log in to Docker Hub
41+
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # 3.4.0
42+
with:
43+
username: ${{ secrets.DOCKERHUB_USERNAME }}
44+
password: ${{ secrets.DOCKERHUB_TOKEN }}
45+
46+
- name: Build and Push Image
47+
id: build-push
48+
uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # 6.15.0
49+
with:
50+
context: apps/distroless/core
51+
file: apps/distroless/core/Dockerfile
52+
platforms: linux/amd64,linux/arm64
53+
# push: ${{ github.event_name != 'pull_request' }}
54+
push: true
55+
tags: celestdev/core-builder:latest
56+
cache-from: type=gha # Pull cache from GitHub Actions cache
57+
cache-to: type=gha,mode=max # Push cache to GitHub Actions cache (mode=max includes all layers)
58+
59+
outputs:
60+
# Output the digest of the pushed image
61+
image_digest: ${{ steps.build-push.outputs.digest }}

‎.github/workflows/distroless_dart.yaml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,6 @@ on:
77
- 'apps/distroless/bin/list_sdks.dart'
88
- 'apps/distroless/dart/**'
99
- '.github/workflows/distroless_dart.yaml'
10-
schedule:
11-
# Every day at midnight UTC
12-
- cron: '0 0 * * *'
1310

1411
# Prevent duplicate runs due to Graphite
1512
# https://graphite.dev/docs/troubleshooting#why-are-my-actions-running-twice

‎apps/cli/fixtures/fixtures_test.dart‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,8 @@ class TestRunner {
160160
exe: Platform.resolvedExecutable,
161161
action: PubAction.upgrade,
162162
workingDirectory: projectRoot,
163-
verbose: Platform.environment.containsKey('CI'),
163+
// verbose: Platform.environment.containsKey('CI'),
164+
verbose: false,
164165
).timeout(const Duration(seconds: 30));
165166
if (updateGoldens) {
166167
if (goldensDir.existsSync()) {

‎apps/cli/lib/src/codegen/api/dockerfile_generator.dart‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
import 'package:celest_ast/celest_ast.dart' as ast;
22
import 'package:celest_ast/celest_ast.dart';
33
import 'package:celest_cli/src/utils/error.dart';
4+
import 'package:celest_cloud/src/proto.dart' as proto;
45
import 'package:mustache_template/mustache_template.dart';
56

67
/// Generates a `Dockerfile` for the user's project so they can self-host it.
78
final class DockerfileGenerator {
8-
DockerfileGenerator({required this.project});
9+
DockerfileGenerator({
10+
required this.assetType,
11+
required this.project,
12+
});
913

14+
final proto.ProjectAsset_Type assetType;
1015
final ast.ResolvedProject project;
1116

1217
static final Template _dartTemplate = Template(r'''
@@ -28,6 +33,19 @@ ENV PORT=8080
2833
EXPOSE $PORT
2934
''');
3035

36+
// TODO(dnys1): Remove `--platform=linux/amd64` when Celest supports arm64.
37+
static const String _dartExeTemplate = r'''
38+
# syntax=docker/dockerfile:1.2
39+
FROM --platform=linux/amd64 celestdev/core-runtime:latest
40+
41+
WORKDIR /app
42+
COPY --chmod=755 main.exe .
43+
COPY celest.json .
44+
45+
ENV PORT=8080
46+
EXPOSE $PORT
47+
''';
48+
3149
// TODO(dnys1): Remove `--platform=linux/amd64` when Celest supports arm64.
3250
static final Template _flutterTemplate = Template(r'''
3351
# syntax=docker/dockerfile:1
@@ -50,6 +68,9 @@ EXPOSE $PORT
5068
''');
5169

5270
String generate() {
71+
if (assetType == proto.ProjectAsset_Type.DART_EXECUTABLE) {
72+
return _dartExeTemplate;
73+
}
5374
return switch (project.sdkConfig.targetSdk) {
5475
SdkType.flutter => _flutterTemplate.renderString({
5576
'version': project.sdkConfig.flutter!.version.canonicalizedVersion,

‎apps/cli/lib/src/compiler/api/entrypoint_compiler.dart‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import 'package:celest_cli/src/context.dart';
99
import 'package:celest_cli/src/sdk/dart_sdk.dart';
1010
import 'package:celest_cli/src/utils/error.dart';
1111
import 'package:celest_cli/src/utils/json.dart';
12+
import 'package:celest_cloud/src/proto.dart' as proto;
1213
import 'package:crypto/crypto.dart';
1314
import 'package:logging/logging.dart';
1415

@@ -31,17 +32,20 @@ final class EntrypointDefinition {
3132

3233
final class EntrypointResult {
3334
const EntrypointResult({
35+
required this.type,
3436
required this.outputDillPath,
3537
required this.outputDill,
3638
required this.outputDillDigest,
3739
});
3840

41+
final proto.ProjectAsset_Type type;
3942
final String outputDillPath;
4043
final Uint8List outputDill;
4144
final Digest outputDillDigest;
4245

4346
@override
4447
String toString() => prettyPrintJson({
48+
'type': type.name,
4549
'outputDillPath': outputDillPath,
4650
'outputDillSha256': outputDillDigest.toString(),
4751
});
@@ -58,6 +62,51 @@ final class EntrypointCompiler {
5862
final bool verbose;
5963
final List<String> enabledExperiments;
6064

65+
Future<EntrypointResult> _crossCompile({
66+
required String entrypointPath,
67+
}) async {
68+
logger.fine('Cross-compiling entrypoint: $entrypointPath');
69+
final outputPath = p.join(p.dirname(entrypointPath), 'main.exe');
70+
final command = <String>[
71+
Sdk.current.dart,
72+
'compile',
73+
'exe',
74+
'--target-os=linux',
75+
'--target-arch=x64',
76+
'--experimental-cross-compilation',
77+
'-o',
78+
outputPath,
79+
entrypointPath,
80+
];
81+
final result = await processManager.run(
82+
command,
83+
workingDirectory: projectPaths.outputsDir,
84+
includeParentEnvironment: true,
85+
stdoutEncoding: utf8,
86+
stderrEncoding: utf8,
87+
);
88+
final ProcessResult(:exitCode, :stdout as String, :stderr as String) =
89+
result;
90+
logger.fine('Cross-compilation finished with status $exitCode');
91+
if (exitCode != 0) {
92+
throw ProcessException(
93+
Sdk.current.dart,
94+
command.sublist(1),
95+
'Cross-compilation failed:\n$stdout\n$stderr',
96+
exitCode,
97+
);
98+
}
99+
100+
final outputDill = await fileSystem.file(outputPath).readAsBytes();
101+
final outputDillDigest = await _computeMd5(outputDill.asUnmodifiableView());
102+
return EntrypointResult(
103+
type: proto.ProjectAsset_Type.DART_EXECUTABLE,
104+
outputDillPath: outputPath,
105+
outputDill: outputDill,
106+
outputDillDigest: outputDillDigest,
107+
);
108+
}
109+
61110
Future<EntrypointResult> compile({
62111
required ast.ResolvedProject resolvedProject,
63112
required String entrypointPath,
@@ -69,16 +118,20 @@ final class EntrypointCompiler {
69118
'$entrypointPath',
70119
);
71120
}
72-
final pathWithoutDart = entrypointPath.substring(
73-
0,
74-
entrypointPath.length - 5,
75-
);
121+
122+
if (resolvedProject.sdkConfig.targetSdk == SdkType.dart &&
123+
Sdk.current.supportsCrossCompilation) {
124+
return _crossCompile(
125+
entrypointPath: entrypointPath,
126+
);
127+
}
128+
76129
final packageConfig = await transformPackageConfig(
77130
packageConfigPath: projectPaths.packagesConfig,
78131
fromRoot: projectPaths.projectRoot,
79132
toRoot: projectPaths.outputsDir,
80133
);
81-
final outputPath = '$pathWithoutDart.dill';
134+
final outputPath = p.join(p.dirname(entrypointPath), 'main.aot.dill');
82135
final (target, platformDill, sdkRoot) =
83136
switch (resolvedProject.sdkConfig.targetSdk) {
84137
SdkType.flutter => (
@@ -136,6 +189,7 @@ final class EntrypointCompiler {
136189
final outputDill = await fileSystem.file(outputPath).readAsBytes();
137190
final outputDillDigest = await _computeMd5(outputDill.asUnmodifiableView());
138191
return EntrypointResult(
192+
type: proto.ProjectAsset_Type.DART_KERNEL,
139193
outputDillPath: outputPath,
140194
outputDill: outputDill,
141195
outputDillDigest: outputDillDigest,

‎apps/cli/lib/src/frontend/celest_frontend.dart‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,7 @@ final class CelestFrontend {
833833
return projectResolver.resolvedProject;
834834
});
835835

836-
Future<void> _writeProjectOutputs({
836+
Future<EntrypointResult> _writeProjectOutputs({
837837
required ResolvedProject resolvedProject,
838838
required String environmentId,
839839
}) async {
@@ -846,13 +846,13 @@ final class CelestFrontend {
846846
verbose: verbose,
847847
enabledExperiments: celestProject.analysisOptions.enabledExperiments,
848848
);
849-
final kernel = await entrypointCompiler.compile(
849+
final output = await entrypointCompiler.compile(
850850
resolvedProject: resolvedProject,
851851
entrypointPath: projectPaths.localApiEntrypoint,
852852
);
853853
await buildOutputs
854-
.childFile('main.aot.dill')
855-
.writeAsBytes(kernel.outputDill);
854+
.childFile(p.basename(output.outputDillPath))
855+
.writeAsBytes(output.outputDill);
856856

857857
// Generate `flutter_assets` for the Flutter app.
858858
if (resolvedProject.sdkConfig.targetSdk == ast.SdkType.flutter) {
@@ -899,14 +899,19 @@ final class CelestFrontend {
899899
}
900900
}
901901

902-
final dockerfile = DockerfileGenerator(project: resolvedProject);
902+
final dockerfile = DockerfileGenerator(
903+
project: resolvedProject,
904+
assetType: output.type,
905+
);
903906
await buildOutputs
904907
.childFile('Dockerfile')
905908
.writeAsString(dockerfile.generate());
906909

907910
await buildOutputs.childFile('celest.json').writeAsString(
908911
prettyPrintJson(resolvedProject.toProto().toProto3Json()),
909912
);
913+
914+
return output;
910915
}
911916

912917
Future<Uri> _startLocalApi(
@@ -1017,27 +1022,24 @@ final class CelestFrontend {
10171022
required ast.ResolvedProject resolvedProject,
10181023
}) =>
10191024
performance.trace('CelestFrontend', 'deployProject', () async {
1020-
await _writeProjectOutputs(
1025+
final output = await _writeProjectOutputs(
10211026
resolvedProject: resolvedProject,
10221027
environmentId: environmentId,
10231028
);
1024-
final (kernelBytes, flutterAssetBytes) = await (
1025-
fileSystem
1026-
.directory(projectPaths.buildDir)
1027-
.childFile('main.aot.dill')
1028-
.readAsBytes(),
1029-
switch (resolvedProject.sdkConfig.targetSdk) {
1029+
final (kernelBytes, flutterAssetBytes) = (
1030+
output.outputDill,
1031+
await switch (resolvedProject.sdkConfig.targetSdk) {
10301032
ast.SdkType.flutter => _tarGzDirectory(
10311033
p.join(projectPaths.buildDir, 'flutter_assets'),
10321034
),
10331035
_ => Future.value(Uint8List(0)),
10341036
},
1035-
).wait;
1037+
);
10361038
final assets = [
10371039
pb.ProjectAsset(
1038-
type: pb.ProjectAsset_Type.DART_KERNEL,
1039-
filename: 'main.aot.dill',
1040-
inline: kernelBytes,
1040+
type: output.type,
1041+
filename: '${p.basenameWithoutExtension(output.outputDillPath)}.gz',
1042+
inline: gzip.encode(kernelBytes),
10411043
),
10421044
if (resolvedProject.sdkConfig.targetSdk == ast.SdkType.flutter)
10431045
pb.ProjectAsset(

‎apps/cli/lib/src/sdk/dart_sdk.dart‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -222,19 +222,13 @@ class Sdk {
222222
'vm_platform_strong_product.dill',
223223
);
224224

225-
/// The version when the new bytecode format and compiler was [introduced](https://github.com/dart-lang/sdk/commit/3abf78212c480cbbbfd43f6382ff262532c90e4d).
226-
///
227-
/// Currently, the SDK does not bundle the `dart2bytecode` tool, so this
228-
/// version just signifies the runtime version that supports the new bytecode
229-
/// format.
230-
static final bytecodeVersion = Version.parse('3.6.0-133.0.dev');
225+
/// The version when cross-compilation was introduced.
226+
static final Version _crossCompilationVersion =
227+
Version.parse('3.8.0-262.0.dev');
231228

232-
/// Whether or not the current SDK supports the new bytecode format.
233-
///
234-
/// Trying to use the `dart2bytecode` tool on an SDK that does not support
235-
/// the new bytecode format will result in an error.
236-
bool get supportsBytecode {
237-
return version >= bytecodeVersion;
229+
/// Whether or not the current SDK supports cross-compilation.
230+
bool get supportsCrossCompilation {
231+
return version >= _crossCompilationVersion;
238232
}
239233
}
240234

‎apps/distroless/core/Dockerfile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# syntax=docker/dockerfile:1
2+
ARG DEBIAN_VERSION=12
3+
4+
# Builds SQLite from Source
5+
#
6+
# The version shipped with apt is very outdated (3.40.1). This is the only way
7+
# it seems to get anything newer.
8+
FROM debian:${DEBIAN_VERSION}-slim AS sqlite
9+
10+
# Install build dependencies
11+
RUN apt-get update && apt-get install -y \
12+
build-essential \
13+
tcl-dev \
14+
libreadline-dev \
15+
wget \
16+
unzip \
17+
&& rm -rf /var/lib/apt/lists/*
18+
19+
# Set up build directory
20+
WORKDIR /sqlite_build
21+
22+
# Download the latest SQLite source
23+
ARG SQLITE_VERSION=3490100
24+
RUN wget https://www.sqlite.org/2025/sqlite-autoconf-${SQLITE_VERSION}.tar.gz \
25+
&& tar xzf sqlite-autoconf-${SQLITE_VERSION}.tar.gz \
26+
&& cd sqlite-autoconf-${SQLITE_VERSION}
27+
28+
# Build SQLite with default options
29+
WORKDIR /sqlite_build/sqlite-autoconf-${SQLITE_VERSION}
30+
RUN CFLAGS='-DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_JSON1' \
31+
./configure --prefix=/usr \
32+
--enable-threadsafe \
33+
&& make -j$(nproc) \
34+
&& make install DESTDIR=/sqlite_install
35+
36+
# Copy SQLite into a minimal runtime image
37+
FROM gcr.io/distroless/base-debian${DEBIAN_VERSION}
38+
COPY --from=sqlite /sqlite_install /
39+
40+
WORKDIR /tmp
41+
CMD ["/app/main.exe"]

0 commit comments

Comments
 (0)