Skip to content

Commit b23df00

Browse files
committed
Compile SQLite with hooks
1 parent eb6c87e commit b23df00

File tree

8 files changed

+221
-293
lines changed

8 files changed

+221
-293
lines changed

.github/workflows/compile_sqlite.yml

Lines changed: 8 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ jobs:
1616
- uses: actions/cache@v4
1717
id: cache_build
1818
with:
19-
path: out/
20-
key: sqlite-src-${{ hashFiles('tool/sqlite3/') }}
19+
path: sqlite-src/
20+
key: sqlite-src-${{ hashFiles('tool/sqlite3_build_id') }}
2121

2222
- uses: dart-lang/setup-dart@v1
2323
if: steps.cache_build.outputs.cache-hit != 'true'
@@ -33,27 +33,25 @@ jobs:
3333
uses: actions/upload-artifact@v4
3434
with:
3535
name: sqlite3-src
36-
path: out/
36+
path: sqlite-src/
3737
if-no-files-found: error
3838
retention-days: 1
3939

4040
build_sqlite:
4141
needs: [download_sqlite]
4242
strategy:
4343
matrix:
44-
# We only really need this for Ubuntu, but we recommend users run the same
45-
# steps so we better make sure they work on all platforms.
46-
os: [ubuntu-latest, macos-latest]
44+
os: [ubuntu-latest, macos-latest, windows-latest]
4745

4846
runs-on: ${{ matrix.os }}
4947
name: Compile sqlite3
5048
steps:
51-
- uses: actions/checkout@v4
49+
- uses: actions/checkout@v5
5250
- uses: actions/cache@v4
5351
id: cache_build
5452
with:
5553
path: out/
56-
key: sqlite-prebuilt-${{ runner.os }}-${{ hashFiles('tool/sqlite3/') }}
54+
key: sqlite-prebuilt-${{ runner.os }}-${{ hashFiles('tool/sqlite3_build_id') }}
5755

5856
- name: Download sqlite3 sources
5957
if: steps.cache_build.outputs.cache-hit != 'true'
@@ -67,12 +65,6 @@ jobs:
6765
with:
6866
sdk: beta
6967

70-
- name: Install cross-compiling GCC
71-
shell: bash
72-
if: runner.os == 'Linux' && steps.cache_build.outputs.cache-hit != 'true'
73-
run: |
74-
sudo apt install -y gcc-aarch64-linux-gnu gcc-riscv64-linux-gnu gcc-arm-linux-gnueabihf gcc-i686-linux-gnu
75-
7668
- name: Compile sqlite3
7769
if: steps.cache_build.outputs.cache-hit != 'true'
7870
run: |
@@ -83,54 +75,7 @@ jobs:
8375
uses: actions/upload-artifact@v4
8476
with:
8577
name: sqlite3-libs-${{ runner.os }}
86-
path: out/
87-
if-no-files-found: error
88-
retention-days: 1
89-
90-
build_sqlite_windows:
91-
needs: [download_sqlite]
92-
strategy:
93-
matrix:
94-
arch: [amd64, amd64_x86, amd64_arm64]
95-
96-
runs-on: windows-latest
97-
name: Compile sqlite3 Windows
98-
steps:
99-
- uses: actions/checkout@v4
100-
- uses: actions/cache@v4
101-
id: cache_build
102-
with:
103-
path: out/
104-
key: sqlite-prebuilt-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('tool/sqlite3/') }}
105-
106-
- name: Download sqlite3 sources
107-
if: steps.cache_build.outputs.cache-hit != 'true'
108-
uses: actions/download-artifact@v4
109-
with:
110-
name: sqlite3-src
111-
path: sqlite3-src
112-
113-
- uses: dart-lang/setup-dart@v1
114-
if: steps.cache_build.outputs.cache-hit != 'true'
115-
with:
116-
sdk: beta
117-
118-
- uses: ilammy/msvc-dev-cmd@v1
119-
if: steps.cache_build.outputs.cache-hit != 'true'
120-
with:
121-
arch: ${{ matrix.arch }}
122-
123-
- name: Compile sqlite3
124-
if: steps.cache_build.outputs.cache-hit != 'true'
125-
run: |
126-
dart pub get
127-
dart run tool/sqlite3/compile_sqlite.dart ${{ matrix.arch }}
128-
129-
- name: Upload sqlite3 binaries
130-
uses: actions/upload-artifact@v4
131-
with:
132-
name: sqlite3-libs-${{ runner.os }}-${{ matrix.arch }}
133-
path: out/*.dll
78+
path: sqlite-compiled
13479
if-no-files-found: error
13580
retention-days: 1
13681

@@ -142,7 +87,7 @@ jobs:
14287
id: cache_build
14388
with:
14489
path: sqlite3_wasm_build/out/
145-
key: sqlite-prebuilt-wasm2-${{ hashFiles('sqlite3_wasm_build') }}
90+
key: sqlite-prebuilt-wasm2-${{ hashFiles('tool/sqlite3_build_id') }}
14691

14792
- name: Download WASI SDK
14893
if: steps.cache_build.outputs.cache-hit != 'true'

pubspec.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ dependencies:
99
crypto: ^3.0.6
1010
pool: ^1.5.2
1111
collection: ^1.19.1
12+
hooks_runner: ^0.23.2
13+
logging: ^1.3.0
14+
file: ^7.0.1
15+
package_config: ^2.2.0
1216

1317
workspace:
1418
- sqlite3

sqlite3/hook/build.dart

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import 'dart:io';
22

33
import 'package:code_assets/code_assets.dart';
44
import 'package:hooks/hooks.dart';
5+
import 'package:native_toolchain_c/native_toolchain_c.dart';
56
import 'package:path/path.dart' as p;
67

78
import 'package:sqlite3/src/hook/description.dart';
@@ -42,9 +43,22 @@ void main(List<String> args) async {
4243
file: target.uri,
4344
),
4445
);
45-
case CompileSqlite():
46-
// TODO: Handle this case.
47-
throw UnimplementedError();
46+
case CompileSqlite(:final sourceFile, :final defines):
47+
final library = CBuilder.library(
48+
name: 'sqlite3',
49+
packageName: 'sqlite3',
50+
assetName: name,
51+
sources: [sourceFile],
52+
includes: [p.dirname(sourceFile)],
53+
defines: defines,
54+
libraries: [
55+
if (input.config.code.targetOS == OS.android)
56+
// We need to link the math library on Android.
57+
'm',
58+
],
59+
);
60+
61+
await library.run(input: input, output: output);
4862
case SimpleBinary():
4963
output.assets.code.add(
5064
CodeAsset(

sqlite3/lib/src/hook/description.dart

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ sealed class SqliteBinary {
3131
return SimpleBinary.fromProcess;
3232
case 'executable':
3333
return SimpleBinary.fromExecutable;
34+
case 'source':
35+
return CompileSqlite(
36+
sourceFile: userDefines.path('path')!.toFilePath(),
37+
defines: CompilerDefines.parse(
38+
userDefines,
39+
input.config.code.targetOS,
40+
),
41+
);
3442
default:
3543
throw ArgumentError.value(
3644
userDefines['source'],
@@ -204,13 +212,13 @@ final class PrecompiledForTesting extends PrecompiledBinary {
204212
}
205213

206214
final class CompileSqlite implements SqliteBinary {
207-
/// How to obtain the `sqlite3.c` source to compile.
208-
final SqliteSources sources;
215+
/// Path to the `sqlite3.c` source file to compile.
216+
final String sourceFile;
209217

210218
/// User-defines for the SQLite compilation.
211219
final CompilerDefines defines;
212220

213-
CompileSqlite({required this.sources, required this.defines});
221+
CompileSqlite({required this.sourceFile, required this.defines});
214222
}
215223

216224
/// If we're compiling SQLite from source, a way to obtain these sources.
@@ -248,15 +256,6 @@ final class DownloadAmalgamation implements SqliteSources {
248256
}
249257
}
250258

251-
/// Compile SQLite from an existing `sqlite3.c` file that has been vendored into
252-
/// the project.
253-
final class ExistingAmalgamation implements SqliteSources {
254-
/// Path to the `sqlite3.c` source file to compile.
255-
final String sqliteSource;
256-
257-
const ExistingAmalgamation(this.sqliteSource);
258-
}
259-
260259
/// Definition options to use when compiling SQLite.
261260
extension type const CompilerDefines(Map<String, String?> flags)
262261
implements Map<String, String?> {

tool/build_sqlite.dart

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
import 'dart:io';
2+
import 'dart:isolate';
3+
4+
import 'package:code_assets/code_assets.dart';
5+
import 'package:file/local.dart';
6+
import 'package:hooks_runner/hooks_runner.dart';
7+
import 'package:logging/logging.dart';
8+
import 'package:package_config/package_config.dart';
9+
import 'package:pool/pool.dart';
10+
11+
final _compileTasks = Pool(Platform.numberOfProcessors);
12+
13+
/// Invokes `package:sqlite3` build hooks for multiple operating systems and
14+
/// architectures, merging outputs into `sqlite3-compiled/`.
15+
void main(List<String> args) async {
16+
hierarchicalLoggingEnabled = true;
17+
Logger('sqlite3').onRecord.listen((record) {
18+
print('[${record.loggerName}]: ${record.message}');
19+
});
20+
21+
var operatingSystems = args.map(OS.fromString).toList();
22+
if (operatingSystems.isEmpty) {
23+
if (Platform.isLinux) {
24+
operatingSystems = [OS.linux, OS.android];
25+
} else if (Platform.isMacOS) {
26+
operatingSystems = [OS.macOS];
27+
} else if (Platform.isWindows) {
28+
operatingSystems = [OS.windows];
29+
}
30+
}
31+
32+
if (operatingSystems.isEmpty) {
33+
print('Usage: dart run tool/build_sqlite3.dart <operating systems...>');
34+
exit(1);
35+
}
36+
print('Compiling for $operatingSystems');
37+
38+
const fs = LocalFileSystem();
39+
40+
final outputDirectory = fs.directory('sqlite-compiled');
41+
print('Compiling to ${outputDirectory.path}');
42+
43+
if (await outputDirectory.exists()) {
44+
await outputDirectory.delete(recursive: true);
45+
}
46+
await fs.directory('sqlite-compiled').create();
47+
48+
final config = await Isolate.packageConfig;
49+
final packageLayout = PackageLayout.fromPackageConfig(
50+
fs,
51+
await loadPackageConfigUri(config!),
52+
config,
53+
'sqlite3',
54+
includeDevDependencies: false,
55+
);
56+
final definesDir = await fs.systemTempDirectory.createTemp('sqlite3-build');
57+
final compilationTasks = <Future<void>>[];
58+
59+
for (final mode in ['sqlite3', 'sqlite3mc']) {
60+
final sourcePath = fs.currentDirectory
61+
.childDirectory('sqlite-src')
62+
.childDirectory(mode)
63+
.childFile(mode == 'sqlite3' ? 'sqlite3.c' : 'sqlite3mc_amalgamation.c')
64+
.absolute
65+
.path;
66+
67+
final fakePubspec = definesDir.childFile('defines_$mode.pubspec.yaml');
68+
await fakePubspec.writeAsString('''
69+
hooks:
70+
user_defines:
71+
sqlite3:
72+
source: source
73+
path: $sourcePath
74+
''');
75+
76+
Future<void> buildAndCopy(OS os, Architecture architecture,
77+
{IOSCodeConfig? iOS, String? osNameOverride}) async {
78+
final osName = osNameOverride ?? os.name;
79+
final runner = NativeAssetsBuildRunner(
80+
logger: Logger('sqlite3.$mode.${osName}.${architecture.name}'),
81+
dartExecutable: Uri.file(Platform.executable),
82+
fileSystem: fs,
83+
packageLayout: packageLayout,
84+
userDefines: UserDefines(workspacePubspec: fakePubspec.uri),
85+
);
86+
87+
final name = os.dylibFileName('${mode}.${architecture.name}.${osName}');
88+
final result = await runner.build(
89+
extensions: [
90+
CodeAssetExtension(
91+
targetArchitecture: Architecture.x64,
92+
targetOS: OS.linux,
93+
linkModePreference: LinkModePreference.dynamic,
94+
iOS: iOS,
95+
)
96+
],
97+
linkingEnabled: false,
98+
);
99+
100+
if (result.isFailure) {
101+
throw result.asFailure.value;
102+
}
103+
104+
final output = result.asSuccess.value;
105+
for (final file in output.encodedAssets) {
106+
if (file.isCodeAsset) {
107+
final code = file.asCodeAsset;
108+
await fs.file(code.file!).copy(outputDirectory.childFile(name).path);
109+
}
110+
}
111+
}
112+
113+
for (final os in operatingSystems) {
114+
for (final architecture in _osToAbis[os]!) {
115+
compilationTasks.add(
116+
_compileTasks.withResource(() => buildAndCopy(os, architecture)));
117+
}
118+
119+
if (os == OS.iOS) {
120+
// Also compile for iOS simulators
121+
final simulatorConfig =
122+
IOSCodeConfig(targetSdk: IOSSdk.iPhoneSimulator, targetVersion: 13);
123+
124+
compilationTasks.add(_compileTasks.withResource(() => buildAndCopy(
125+
os, Architecture.arm64,
126+
iOS: simulatorConfig, osNameOverride: 'ios_sim')));
127+
compilationTasks.add(_compileTasks.withResource(() => buildAndCopy(
128+
os, Architecture.x64,
129+
iOS: simulatorConfig, osNameOverride: 'ios_sim')));
130+
}
131+
}
132+
}
133+
134+
await Future.wait(compilationTasks, eagerError: true);
135+
print('Done building');
136+
await definesDir.delete(recursive: true);
137+
}
138+
139+
const _osToAbis = {
140+
OS.linux: [
141+
Architecture.arm,
142+
Architecture.arm64,
143+
Architecture.ia32,
144+
Architecture.x64,
145+
Architecture.riscv64,
146+
],
147+
OS.android: [
148+
Architecture.arm,
149+
Architecture.arm64,
150+
Architecture.ia32,
151+
Architecture.x64,
152+
],
153+
OS.windows: [
154+
Architecture.arm64,
155+
Architecture.ia32,
156+
Architecture.x64,
157+
],
158+
OS.macOS: [
159+
Architecture.arm64,
160+
Architecture.x64,
161+
],
162+
OS.iOS: [
163+
Architecture.arm64,
164+
// Note: There's a special check to also compile simulator builds for x64
165+
],
166+
};

0 commit comments

Comments
 (0)