Skip to content

Commit 0d4ae21

Browse files
nickmeinholdclaude
nickmeinhold
and
claude
authored
feat(crdt): schema v2 with is_deleted tombstone + HLC stamping (#41) (#121)
* feat(crdt): schema v2 with is_deleted tombstone + HLC stamping on writes (#41) Add `is_deleted` boolean column to all 6 Drift tables for CRDT tombstone support. Wire HlcManager into DriftGraphRepository so every write (save, updateQuizItem, saveSplitData) is stamped with an HLC timestamp. load() now filters out tombstoned rows. Schema migration from v1→v2 uses ALTER TABLE ADD COLUMN. No external behavior changes — HlcManager is optional (backward compatible with existing callers). 7 new tests: HLC stamping verification + isDeleted filtering across all entity types. Full suite: 855/855 pass. Co-Authored-By: Claude <noreply@anthropic.com> * fix(crdt): address review — TODO comment, batch HLC comment, migration tests - Add TODO(#41) on DELETE ALL noting PR 3 will replace with upsert + tombstone - Add comment explaining single HLC per batch is intentional (one event = one HLC) - Add 3 migration tests: is_deleted defaults to false, column exists on all tables, tombstone can be set to true Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
1 parent 75c7019 commit 0d4ae21

File tree

7 files changed

+831
-44
lines changed

7 files changed

+831
-44
lines changed

‎lib/src/providers/graph_store_provider.dart‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import '../storage/dual_write_graph_repository.dart';
66
import '../storage/firestore_graph_repository.dart';
77
import '../storage/graph_repository.dart';
88
import 'auth_provider.dart';
9+
import 'hlc_provider.dart';
910

1011
/// Provides the singleton [EngramDatabase] instance.
1112
///
@@ -28,7 +29,8 @@ final engramDatabaseProvider = Provider<EngramDatabase>(
2829
final graphRepositoryProvider = Provider<GraphRepository>((ref) {
2930
final user = ref.watch(authStateProvider).valueOrNull;
3031
final db = ref.watch(engramDatabaseProvider);
31-
final driftRepo = DriftGraphRepository(db: db);
32+
final hlcManager = ref.watch(hlcManagerProvider);
33+
final driftRepo = DriftGraphRepository(db: db, hlcManager: hlcManager);
3234

3335
if (user != null) {
3436
final firestore = ref.watch(firestoreProvider);

‎lib/src/storage/drift/drift_graph_repository.dart‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import 'package:drift/drift.dart';
22

3+
import '../../crdt/hlc_manager.dart';
34
import '../../models/concept.dart';
45
import '../../models/knowledge_graph.dart';
56
import '../../models/quiz_item.dart';
@@ -18,25 +19,49 @@ import 'engram_database.dart';
1819
/// atomic updates. The `save` method uses DELETE ALL + INSERT ALL — this
1920
/// is simpler than Firestore's upsert+diff pattern because SQLite
2021
/// transactions are local and fast.
22+
///
23+
/// When an [HlcManager] is provided, every write is stamped with a
24+
/// Hybrid Logical Clock timestamp for CRDT sync (#41). Rows with
25+
/// `isDeleted = true` are excluded from [load] results but preserved
26+
/// in the database for changeset propagation.
2127
class DriftGraphRepository extends GraphRepository {
22-
DriftGraphRepository({required EngramDatabase db}) : _db = db;
28+
DriftGraphRepository({required EngramDatabase db, HlcManager? hlcManager})
29+
: _db = db,
30+
_hlcManager = hlcManager;
2331

2432
final EngramDatabase _db;
33+
final HlcManager? _hlcManager;
34+
35+
/// Returns the current HLC string for stamping writes, or empty string
36+
/// if no HlcManager is configured (backward compatible).
37+
String _stampHlc() => _hlcManager?.now().toString() ?? '';
2538

2639
// -------------------------------------------------------------------------
2740
// load
2841
// -------------------------------------------------------------------------
2942

3043
@override
3144
Future<KnowledgeGraph> load() async {
32-
// Query all tables in parallel.
45+
// Query all tables in parallel, filtering out tombstoned rows.
3346
final results = await Future.wait([
34-
_db.select(_db.driftConcepts).get(), // 0
35-
_db.select(_db.driftRelationships).get(), // 1
36-
_db.select(_db.driftQuizItems).get(), // 2
37-
_db.select(_db.driftDocuments).get(), // 3
38-
_db.select(_db.driftTopics).get(), // 4
39-
_db.select(_db.driftTopicDocuments).get(), // 5
47+
(_db.select(_db.driftConcepts)
48+
..where((t) => t.isDeleted.equals(false)))
49+
.get(), // 0
50+
(_db.select(_db.driftRelationships)
51+
..where((t) => t.isDeleted.equals(false)))
52+
.get(), // 1
53+
(_db.select(_db.driftQuizItems)
54+
..where((t) => t.isDeleted.equals(false)))
55+
.get(), // 2
56+
(_db.select(_db.driftDocuments)
57+
..where((t) => t.isDeleted.equals(false)))
58+
.get(), // 3
59+
(_db.select(_db.driftTopics)
60+
..where((t) => t.isDeleted.equals(false)))
61+
.get(), // 4
62+
(_db.select(_db.driftTopicDocuments)
63+
..where((t) => t.isDeleted.equals(false)))
64+
.get(), // 5
4065
]);
4166

4267
final conceptRows = results[0] as List<DriftConcept>;
@@ -74,6 +99,9 @@ class DriftGraphRepository extends GraphRepository {
7499
Future<void> save(KnowledgeGraph graph) async {
75100
await _db.transaction(() async {
76101
// 1. Delete all existing data.
102+
// TODO(#41): Replace DELETE ALL with upsert + orphan tombstoning in PR 3.
103+
// Currently this wipes tombstones — acceptable while save() is the only
104+
// write path, but must change before changeset sync is enabled.
77105
await Future.wait([
78106
_db.delete(_db.driftTopicDocuments).go(),
79107
_db.delete(_db.driftTopics).go(),
@@ -85,31 +113,37 @@ class DriftGraphRepository extends GraphRepository {
85113

86114
// 2. Batch-insert all entities. Uses insertOrReplace as a safety net
87115
// in case the model layer has duplicate IDs (e.g. cross-document
88-
// concept reuse during extraction).
116+
// concept reuse during extraction). Each row is stamped with an
117+
// HLC timestamp for CRDT sync.
118+
//
119+
// A single HLC is used for the entire batch — intentional. All rows
120+
// in an atomic save share the same causal timestamp, which is correct
121+
// for CRDT semantics (one logical event = one HLC).
122+
final hlc = _stampHlc();
89123
await _db.batch((batch) {
90124
batch.insertAll(
91125
_db.driftConcepts,
92-
graph.concepts.map((c) => c.toCompanion()).toList(),
126+
graph.concepts.map((c) => c.toCompanion(hlc: hlc)).toList(),
93127
mode: InsertMode.insertOrReplace,
94128
);
95129
batch.insertAll(
96130
_db.driftRelationships,
97-
graph.relationships.map((r) => r.toCompanion()).toList(),
131+
graph.relationships.map((r) => r.toCompanion(hlc: hlc)).toList(),
98132
mode: InsertMode.insertOrReplace,
99133
);
100134
batch.insertAll(
101135
_db.driftQuizItems,
102-
graph.quizItems.map((q) => q.toCompanion()).toList(),
136+
graph.quizItems.map((q) => q.toCompanion(hlc: hlc)).toList(),
103137
mode: InsertMode.insertOrReplace,
104138
);
105139
batch.insertAll(
106140
_db.driftDocuments,
107-
graph.documentMetadata.map((d) => d.toCompanion()).toList(),
141+
graph.documentMetadata.map((d) => d.toCompanion(hlc: hlc)).toList(),
108142
mode: InsertMode.insertOrReplace,
109143
);
110144
batch.insertAll(
111145
_db.driftTopics,
112-
graph.topics.map((t) => t.toCompanion()).toList(),
146+
graph.topics.map((t) => t.toCompanion(hlc: hlc)).toList(),
113147
mode: InsertMode.insertOrReplace,
114148
);
115149

@@ -121,6 +155,7 @@ class DriftGraphRepository extends GraphRepository {
121155
DriftTopicDocumentsCompanion.insert(
122156
topicId: topic.id,
123157
documentId: docId,
158+
hlc: Value(hlc),
124159
),
125160
mode: InsertMode.insertOrReplace,
126161
);
@@ -138,7 +173,7 @@ class DriftGraphRepository extends GraphRepository {
138173
Future<void> updateQuizItem(KnowledgeGraph graph, QuizItem item) async {
139174
await _db
140175
.into(_db.driftQuizItems)
141-
.insertOnConflictUpdate(item.toCompanion());
176+
.insertOnConflictUpdate(item.toCompanion(hlc: _stampHlc()));
142177
}
143178

144179
// -------------------------------------------------------------------------
@@ -152,20 +187,21 @@ class DriftGraphRepository extends GraphRepository {
152187
required List<Relationship> relationships,
153188
required List<QuizItem> quizItems,
154189
}) async {
190+
final hlc = _stampHlc();
155191
await _db.batch((batch) {
156192
batch.insertAll(
157193
_db.driftConcepts,
158-
concepts.map((c) => c.toCompanion()).toList(),
194+
concepts.map((c) => c.toCompanion(hlc: hlc)).toList(),
159195
mode: InsertMode.insertOrReplace,
160196
);
161197
batch.insertAll(
162198
_db.driftRelationships,
163-
relationships.map((r) => r.toCompanion()).toList(),
199+
relationships.map((r) => r.toCompanion(hlc: hlc)).toList(),
164200
mode: InsertMode.insertOrReplace,
165201
);
166202
batch.insertAll(
167203
_db.driftQuizItems,
168-
quizItems.map((q) => q.toCompanion()).toList(),
204+
quizItems.map((q) => q.toCompanion(hlc: hlc)).toList(),
169205
mode: InsertMode.insertOrReplace,
170206
);
171207
});

‎lib/src/storage/drift/drift_mappers.dart‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@ import 'engram_database.dart';
1313

1414
/// Maps a domain [Concept] to a Drift companion for insertion.
1515
///
16-
/// Omits `embedding` and `hlc` — the database applies defaults (null and ''
17-
/// respectively). These columns are reserved for future features (#39, #41).
16+
/// Pass [hlc] to stamp the row with a CRDT timestamp. If omitted, the
17+
/// database default (empty string) is used. `embedding` is omitted
18+
/// (reserved for #39).
1819
extension ConceptToCompanion on Concept {
19-
DriftConceptsCompanion toCompanion() {
20+
DriftConceptsCompanion toCompanion({String hlc = ''}) {
2021
return DriftConceptsCompanion.insert(
2122
id: id,
2223
name: name,
@@ -27,6 +28,7 @@ extension ConceptToCompanion on Concept {
2728
parentConceptId != null
2829
? Value(parentConceptId)
2930
: const Value.absent(),
31+
hlc: Value(hlc),
3032
);
3133
}
3234
}
@@ -57,7 +59,7 @@ extension DriftConceptToDomain on DriftConcept {
5759
/// Always stores [Relationship.resolvedType] (never null) — the DB column
5860
/// is non-nullable, matching [Relationship.toJson] behaviour.
5961
extension RelationshipToCompanion on Relationship {
60-
DriftRelationshipsCompanion toCompanion() {
62+
DriftRelationshipsCompanion toCompanion({String hlc = ''}) {
6163
return DriftRelationshipsCompanion.insert(
6264
id: id,
6365
fromConceptId: fromConceptId,
@@ -66,6 +68,7 @@ extension RelationshipToCompanion on Relationship {
6668
description:
6769
description != null ? Value(description) : const Value.absent(),
6870
type: resolvedType,
71+
hlc: Value(hlc),
6972
);
7073
}
7174
}
@@ -97,7 +100,7 @@ extension DriftRelationshipToDomain on DriftRelationship {
97100
/// Drift's TEXT columns handle them naturally and it keeps the schema simple
98101
/// for CRDT sync (string comparison works for HLC timestamps too).
99102
extension QuizItemToCompanion on QuizItem {
100-
DriftQuizItemsCompanion toCompanion() {
103+
DriftQuizItemsCompanion toCompanion({String hlc = ''}) {
101104
return DriftQuizItemsCompanion.insert(
102105
id: id,
103106
conceptId: conceptId,
@@ -121,6 +124,7 @@ extension QuizItemToCompanion on QuizItem {
121124
? Value(predictedDifficulty)
122125
: const Value.absent(),
123126
reviewCount: Value(reviewCount),
127+
hlc: Value(hlc),
124128
);
125129
}
126130
}
@@ -159,7 +163,7 @@ extension DriftQuizItemToDomain on DriftQuizItem {
159163
/// passthrough), so it maps directly. [DocumentMetadata.ingestedAt] is a
160164
/// DateTime that we convert to ISO 8601.
161165
extension DocumentMetadataToCompanion on DocumentMetadata {
162-
DriftDocumentsCompanion toCompanion() {
166+
DriftDocumentsCompanion toCompanion({String hlc = ''}) {
163167
return DriftDocumentsCompanion.insert(
164168
documentId: documentId,
165169
title: title,
@@ -173,6 +177,7 @@ extension DocumentMetadataToCompanion on DocumentMetadata {
173177
: const Value.absent(),
174178
ingestedText:
175179
ingestedText != null ? Value(ingestedText) : const Value.absent(),
180+
hlc: Value(hlc),
176181
);
177182
}
178183
}
@@ -202,7 +207,7 @@ extension DriftDocumentToDomain on DriftDocument {
202207
/// creating [DriftTopicDocumentsCompanion] rows for each document ID in the
203208
/// topic's [Topic.documentIds] set.
204209
extension TopicToCompanion on Topic {
205-
DriftTopicsCompanion toCompanion() {
210+
DriftTopicsCompanion toCompanion({String hlc = ''}) {
206211
return DriftTopicsCompanion.insert(
207212
id: id,
208213
name: name,
@@ -213,6 +218,7 @@ extension TopicToCompanion on Topic {
213218
lastIngestedAt != null
214219
? Value(lastIngestedAt!.toIso8601String())
215220
: const Value.absent(),
221+
hlc: Value(hlc),
216222
);
217223
}
218224
}

‎lib/src/storage/drift/engram_database.dart‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ part 'engram_database.g.dart';
1616
/// Maps 1:1 to the domain model. `tags` is stored as JSON TEXT since it's
1717
/// never queried independently. `embedding` is reserved for future concept
1818
/// embedding support (#39). `hlc` is the Hybrid Logical Clock timestamp
19-
/// for CRDT sync (#41).
19+
/// for CRDT sync (#41). `isDeleted` is a tombstone flag — deleted rows are
20+
/// hidden from [load] but preserved for changeset propagation.
2021
@TableIndex(name: 'idx_concepts_source_document', columns: {#sourceDocumentId})
2122
class DriftConcepts extends Table {
2223
TextColumn get id => text()();
@@ -27,6 +28,7 @@ class DriftConcepts extends Table {
2728
TextColumn get parentConceptId => text().nullable()();
2829
BlobColumn get embedding => blob().nullable()();
2930
TextColumn get hlc => text().withDefault(const Constant(''))();
31+
BoolColumn get isDeleted => boolean().withDefault(const Constant(false))();
3032

3133
@override
3234
Set<Column> get primaryKey => {id};
@@ -52,6 +54,7 @@ class DriftRelationships extends Table {
5254
TextColumn get type =>
5355
text().map(const RelationshipTypeConverter())();
5456
TextColumn get hlc => text().withDefault(const Constant(''))();
57+
BoolColumn get isDeleted => boolean().withDefault(const Constant(false))();
5558

5659
@override
5760
Set<Column> get primaryKey => {id};
@@ -79,6 +82,7 @@ class DriftQuizItems extends Table {
7982
RealColumn get predictedDifficulty => real().nullable()();
8083
IntColumn get reviewCount => integer().withDefault(const Constant(0))();
8184
TextColumn get hlc => text().withDefault(const Constant(''))();
85+
BoolColumn get isDeleted => boolean().withDefault(const Constant(false))();
8286

8387
@override
8488
Set<Column> get primaryKey => {id};
@@ -97,6 +101,7 @@ class DriftDocuments extends Table {
97101
TextColumn get collectionName => text().nullable()();
98102
TextColumn get ingestedText => text().nullable()();
99103
TextColumn get hlc => text().withDefault(const Constant(''))();
104+
BoolColumn get isDeleted => boolean().withDefault(const Constant(false))();
100105

101106
@override
102107
Set<Column> get primaryKey => {documentId};
@@ -114,6 +119,7 @@ class DriftTopics extends Table {
114119
TextColumn get createdAt => text()();
115120
TextColumn get lastIngestedAt => text().nullable()();
116121
TextColumn get hlc => text().withDefault(const Constant(''))();
122+
BoolColumn get isDeleted => boolean().withDefault(const Constant(false))();
117123

118124
@override
119125
Set<Column> get primaryKey => {id};
@@ -128,6 +134,7 @@ class DriftTopicDocuments extends Table {
128134
TextColumn get topicId => text()();
129135
TextColumn get documentId => text()();
130136
TextColumn get hlc => text().withDefault(const Constant(''))();
137+
BoolColumn get isDeleted => boolean().withDefault(const Constant(false))();
131138

132139
@override
133140
Set<Column> get primaryKey => {topicId, documentId};
@@ -161,7 +168,30 @@ class EngramDatabase extends _$EngramDatabase {
161168
EngramDatabase.forTesting(super.executor);
162169

163170
@override
164-
int get schemaVersion => 1;
171+
int get schemaVersion => 2;
172+
173+
@override
174+
MigrationStrategy get migration => MigrationStrategy(
175+
onCreate: (m) => m.createAll(),
176+
onUpgrade: (m, from, to) async {
177+
if (from < 2) {
178+
// Add is_deleted column with default false to all 6 tables.
179+
for (final table in [
180+
'drift_concepts',
181+
'drift_relationships',
182+
'drift_quiz_items',
183+
'drift_documents',
184+
'drift_topics',
185+
'drift_topic_documents',
186+
]) {
187+
await m.database.customStatement(
188+
'ALTER TABLE $table ADD COLUMN is_deleted INTEGER '
189+
'NOT NULL DEFAULT 0',
190+
);
191+
}
192+
}
193+
},
194+
);
165195

166196
/// Opens the default on-disk database using drift_flutter.
167197
static QueryExecutor _openDefault() {

0 commit comments

Comments
 (0)