Added count tracking to the datastore

Author: dimitribouniol

README.md

@@ -51,7 +51,6 @@ targets: [
 ### Road to 0.1 Betas
 
 As this project matures towards its first beta, a number of features still need to be fleshed out:
-- Keeping track of the number of entries
 - Update snapshots to have multiple manifests
 - Fleshing out historical edit metadata
 - Migrating entries

Sources/CodableDatastore/Datastore/Datastore.swift

@@ -237,6 +237,18 @@ extension Datastore where AccessMode == ReadWrite {
 // MARK: - Loading
 
 extension Datastore {
+    /// The number of objects in the datastore.
+    ///
+    /// - Note: This count may not reflect an up to dat value while data is being written concurrently, but will be acurate after such a transaction finishes.
+    public var count: Int {
+        get async throws {
+            return try await persistence._withTransaction(options: [.idempotent, .readOnly]) { transaction in
+                let descriptor = try await transaction.datastoreDescriptor(for: self.key)
+                return descriptor?.size ?? 0
+            }
+        }
+    }
+    
     public func load(_ idenfifier: IdentifierType) async throws -> CodedType? {
         try await warmupIfNeeded()
 

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRoot.swift

@@ -139,6 +139,91 @@ extension DiskPersistence.Datastore.RootObject {
 // MARK: - Mutations
 
 extension DiskPersistence.Datastore.RootObject {
+    func manifest(
+        applying descriptor: DatastoreDescriptor
+    ) async throws -> (
+        manifest: DatastoreRootManifest,
+        createdIndexes: Set<DiskPersistence.Datastore.Index>
+        /// Note that indexes are not removed here.
+    ) {
+        let originalManifest = try await manifest
+        var manifest = originalManifest
+        
+        manifest.descriptor.version = descriptor.version
+        manifest.descriptor.codedType = descriptor.codedType
+        manifest.descriptor.identifierType = descriptor.identifierType
+        
+        var createdIndexes: Set<DiskPersistence.Datastore.Index> = []
+        
+        for (_, indexDescriptor) in descriptor.directIndexes {
+            let key = indexDescriptor.key
+            let indexType = indexDescriptor.indexType
+            var version = indexDescriptor.version
+            
+            if let originalVersion = originalManifest.descriptor.directIndexes[key]?.version {
+                version = originalVersion
+            } else {
+                let indexInfo = DatastoreRootManifest.IndexInfo(
+                    key: key,
+                    id: DatastoreIndexIdentifier(name: key),
+                    root: DatastoreIndexManifestIdentifier()
+                )
+                let index = DiskPersistence.Datastore.Index(
+                    datastore: datastore,
+                    id: .direct(index: indexInfo.id, manifest: indexInfo.root),
+                    manifest: DatastoreIndexManifest(
+                        id: indexInfo.root,
+                        orderedPages: []
+                    )
+                )
+                createdIndexes.insert(index)
+            }
+            
+            manifest.descriptor.directIndexes[key] = DatastoreDescriptor.IndexDescriptor(
+                version: version,
+                key: key,
+                indexType: indexType
+            )
+        }
+        
+        for (_, indexDescriptor) in descriptor.secondaryIndexes {
+            let key = indexDescriptor.key
+            let indexType = indexDescriptor.indexType
+            var version = indexDescriptor.version
+            
+            if let originalVersion = originalManifest.descriptor.secondaryIndexes[key]?.version {
+                version = originalVersion
+            } else {
+                let indexInfo = DatastoreRootManifest.IndexInfo(
+                    key: key,
+                    id: DatastoreIndexIdentifier(name: key),
+                    root: DatastoreIndexManifestIdentifier()
+                )
+                let index = DiskPersistence.Datastore.Index(
+                    datastore: datastore,
+                    id: .secondary(index: indexInfo.id, manifest: indexInfo.root),
+                    manifest: DatastoreIndexManifest(
+                        id: indexInfo.root,
+                        orderedPages: []
+                    )
+                )
+                createdIndexes.insert(index)
+            }
+            
+            manifest.descriptor.secondaryIndexes[key] = DatastoreDescriptor.IndexDescriptor(
+                version: version,
+                key: key,
+                indexType: indexType
+            )
+        }
+        
+        if originalManifest.descriptor != manifest.descriptor {
+            manifest.id = DatastoreRootIdentifier()
+            manifest.modificationDate = Date()
+        }
+        return (manifest: manifest, createdIndexes: createdIndexes)
+    }
+    
     func manifest(
         replacing index: DiskPersistence.Datastore.Index.ID
     ) async throws -> DatastoreRootManifest {
@@ -168,6 +253,7 @@ extension DiskPersistence.Datastore.RootObject {
 
         if manifest != updatedManifest {
             updatedManifest.id = DatastoreRootIdentifier()
+            updatedManifest.modificationDate = Date()
         }
         return updatedManifest
     }

Sources/CodableDatastore/Persistence/Disk Persistence/Transaction/Transaction.swift

@@ -265,27 +265,28 @@ extension DiskPersistence.Transaction: DatastoreInterfaceProtocol {
     func apply(descriptor: DatastoreDescriptor, for datastoreKey: DatastoreKey) async throws {
         try checkIsActive()
 
-        if let rootObject = try await rootObject(for: datastoreKey) {
-            var manifest = try await rootObject.manifest
+        if let existingRootObject = try await rootObject(for: datastoreKey) {
+            let datastore = existingRootObject.datastore
 
-            // TODO: Do a better merge of these descriptors, especially since size is something we want to preserve, amongst other properties
-            guard manifest.descriptor != descriptor else { return }
+            let (rootManifest, newIndexes) = try await existingRootObject.manifest(applying: descriptor)
 
-            manifest.id = DatastoreRootIdentifier()
-            manifest.modificationDate = Date()
-            manifest.descriptor = descriptor
+            /// No change occured, bail early
+            guard existingRootObject.id != rootManifest.id else { return }
 
-            let newRoot = DiskPersistence.Datastore.RootObject(
-                datastore: rootObject.datastore,
-                id: manifest.id,
-                rootObject: manifest
-            )
-            rootObjects[datastoreKey] = newRoot
-            createdRootObjects.insert(newRoot)
-            deletedRootObjects.insert(rootObject)
-            await newRoot.datastore.adopt(rootObject: newRoot)
+            for newIndex in newIndexes {
+                createdIndexes.insert(newIndex)
+                await datastore.adopt(index: newIndex)
+            }
 
-            // TODO: Don't forget to create the new index objects too!
+            let newRootObject = DiskPersistence.Datastore.RootObject(
+                datastore: datastore,
+                id: rootManifest.id,
+                rootObject: rootManifest
+            )
+            createdRootObjects.insert(newRootObject)
+            deletedRootObjects.insert(existingRootObject)
+            await datastore.adopt(rootObject: newRootObject)
+            rootObjects[datastoreKey] = newRootObject
         } else {
             let (datastore, _) = try await persistence.persistenceDatastore(for: datastoreKey)
 
@@ -345,6 +346,9 @@ extension DiskPersistence.Transaction: DatastoreInterfaceProtocol {
                 await datastore.adopt(index: index)
             }
 
+            var descriptor = descriptor
+            descriptor.size = 0
+            
             /// Create the root object from the indexes that were created
             let manifest = DatastoreRootManifest(
                 id: DatastoreRootIdentifier(),
@@ -555,12 +559,13 @@ extension DiskPersistence.Transaction {
 
         let datastore = existingRootObject.datastore
 
-        let (indexManifest, newPages, removedPages) = try await {
+        /// Depending on the cursor type, insert or replace the entry in the index, capturing the new manifesr, added and removed pages, and change in the number of entries.
+        let ((indexManifest, newPages, removedPages), newEntryCount) = try await {
             switch try cursor(for: someCursor) {
             case .insertion(let cursor):
-                return try await existingIndex.manifest(inserting: entry, at: cursor)
+                return (try await existingIndex.manifest(inserting: entry, at: cursor), 1)
             case .instance(let cursor):
-                return try await existingIndex.manifest(replacing: entry, at: cursor)
+                return (try await existingIndex.manifest(replacing: entry, at: cursor), 0)
             }
         }()
 
@@ -582,7 +587,12 @@ extension DiskPersistence.Transaction {
         deletedIndexes.insert(existingIndex)
         await datastore.adopt(index: newIndex)
 
-        let rootManifest = try await existingRootObject.manifest(replacing: newIndex.id)
+        var rootManifest = try await existingRootObject.manifest(replacing: newIndex.id)
+        
+        /// If the index we are modifying is the primary one, update the number of entries we are managing.
+        if case .primary = newIndex.id {
+            rootManifest.descriptor.size += newEntryCount
+        }
 
         /// No change occured, bail early
         guard existingRootObject.id != rootManifest.id else { return }
@@ -652,16 +662,21 @@ extension DiskPersistence.Transaction {
         }
         deletedPages.formUnion(removedPages)
 
-        let newPrimaryIndex = DiskPersistence.Datastore.Index(
+        let newIndex = DiskPersistence.Datastore.Index(
             datastore: datastore,
             id: existingIndex.id.with(manifestID: indexManifest.id),
             manifest: indexManifest
         )
-        createdIndexes.insert(newPrimaryIndex)
+        createdIndexes.insert(newIndex)
         deletedIndexes.insert(existingIndex)
-        await datastore.adopt(index: newPrimaryIndex)
+        await datastore.adopt(index: newIndex)
 
-        let rootManifest = try await existingRootObject.manifest(replacing: newPrimaryIndex.id)
+        var rootManifest = try await existingRootObject.manifest(replacing: newIndex.id)
+        
+        /// If the index we are modifying is the primary one, update the number of entries we are managing.
+        if case .primary = newIndex.id {
+            rootManifest.descriptor.size -= 1
+        }
 
         /// No change occured, bail early
         guard existingRootObject.id != rootManifest.id else { return }

Tests/CodableDatastoreTests/DiskPersistenceDatastoreTests.swift

@@ -46,6 +46,9 @@ final class DiskPersistenceDatastoreTests: XCTestCase {
         try await datastore.persist(TestStruct(id: "3", value: "My name is Dimitri"))
         try await datastore.persist(TestStruct(id: "1", value: "Hello, World!"))
         try await datastore.persist(TestStruct(id: "2", value: "Twenty Three is Number One"))
+        
+        let count = try await datastore.count
+        XCTAssertEqual(count, 3)
     }
 
     func testWritingEntryWithIndex() async throws {
@@ -74,6 +77,9 @@ final class DiskPersistenceDatastoreTests: XCTestCase {
         try await datastore.persist(TestStruct(id: "3", value: "My name is Dimitri"))
         try await datastore.persist(TestStruct(id: "1", value: "Hello, World!"))
         try await datastore.persist(TestStruct(id: "2", value: "Twenty Three is Number One"))
+        
+        let count = try await datastore.count
+        XCTAssertEqual(count, 3)
     }
 
     func testWritingManyEntries() async throws {