data store (#286)

* data store

* fix

* update delay

* fix

Author: xlc

.swiftformat

@@ -2,3 +2,4 @@
 --maxwidth 140
 --asynccapturing debugCheck
 --closurevoid preserve
+--disable typeSugar

Blockchain/Sources/Blockchain/AuxDataStore/DataStore.swift

@@ -0,0 +1,47 @@
+import Utils
+
+enum DataStoreError: Error {
+    case invalidPackageHash(Data32)
+    case invalidSegmentRoot(Data32)
+}
+
+public final class DataStore: Sendable {
+    private let impl: DataStoreProtocol
+    private let network: DataStoreNetworkProtocol
+
+    public init(_ impl: DataStoreProtocol, _ network: DataStoreNetworkProtocol) {
+        self.impl = impl
+        self.network = network
+    }
+
+    public func fetchSegment(segments: [WorkItem.ImportedDataSegment]) async throws -> [Data4104] {
+        var result: [Data4104] = []
+
+        for segment in segments {
+            let segmentRoot = switch segment.root {
+            case let .segmentRoot(root):
+                root
+            case let .workPackageHash(hash):
+                try await impl.getSegmentRoot(forWorkPackageHash: hash).unwrap(orError: DataStoreError.invalidPackageHash(hash))
+            }
+            let erasureRoot = try await impl.getEasureRoot(forSegmentRoot: segmentRoot)
+                .unwrap(orError: DataStoreError.invalidSegmentRoot(segmentRoot))
+
+            if let localData = try await impl.get(erasureRoot: erasureRoot, index: segment.index) {
+                result.append(localData)
+            } else {
+                // TODO: use network for fetch shards and reconstruct the segment
+                fatalError("not implemented")
+            }
+        }
+
+        return result
+    }
+
+    public func set(data: Data4104, erasureRoot: Data32, index: UInt16) async throws {
+        try await impl.set(data: data, erasureRoot: erasureRoot, index: index)
+
+        // TODO; erasure code the data and store each chunk
+        // so assurer can query them later with CE137
+    }
+}

Blockchain/Sources/Blockchain/AuxDataStore/DataStoreProtocol.swift

@@ -0,0 +1,22 @@
+import Utils
+
+public protocol DataStoreProtocol: Sendable {
+    // segment root => erasure root
+    func getEasureRoot(forSegmentRoot: Data32) async throws -> Data32?
+    func set(erasureRoot: Data32, forSegmentRoot: Data32) async throws
+    func delete(erasureRoot: Data32) async throws
+
+    // work package hash => segment root
+    func getSegmentRoot(forWorkPackageHash: Data32) async throws -> Data32?
+    func set(segmentRoot: Data32, forWorkPackageHash: Data32) async throws
+    func delete(segmentRoot: Data32) async throws
+
+    // erasure root + index => segment data
+    func get(erasureRoot: Data32, index: UInt16) async throws -> Data4104?
+    func set(data: Data4104, erasureRoot: Data32, index: UInt16) async throws
+}
+
+public protocol DataStoreNetworkProtocol: Sendable {
+    // Use CE139/CE140 to fetch remote chunk
+    func fetchRemoteChunk(erasureRoot: Data32, shardIndex: UInt16, segmentIndices: [UInt16]) async throws -> Data12?
+}

Blockchain/Sources/Blockchain/AuxDataStore/InMemoryDataStoreBackend.swift

@@ -0,0 +1,54 @@
+import Utils
+
+public actor InMemoryDataStoreBackend {
+    // segment root => erasure root
+    private var erasureRootBySegmentRoot: [Data32: Data32] = [:]
+
+    // work package hash => segment root
+    private var segmentRootByWorkPackageHash: [Data32: Data32] = [:]
+
+    // erasure root + index => segment data
+    private var chunks: [Data32: [UInt16: Data4104]] = [:]
+
+    public init() {}
+}
+
+extension InMemoryDataStoreBackend: DataStoreProtocol {
+    public func getEasureRoot(forSegmentRoot: Data32) async throws -> Data32? {
+        erasureRootBySegmentRoot[forSegmentRoot]
+    }
+
+    public func set(erasureRoot: Data32, forSegmentRoot: Data32) async throws {
+        erasureRootBySegmentRoot[forSegmentRoot] = erasureRoot
+    }
+
+    public func delete(erasureRoot: Data32) async throws {
+        erasureRootBySegmentRoot.removeValue(forKey: erasureRoot)
+    }
+
+    public func getSegmentRoot(forWorkPackageHash: Data32) async throws -> Data32? {
+        segmentRootByWorkPackageHash[forWorkPackageHash]
+    }
+
+    public func set(segmentRoot: Data32, forWorkPackageHash: Data32) async throws {
+        segmentRootByWorkPackageHash[forWorkPackageHash] = segmentRoot
+    }
+
+    public func delete(segmentRoot: Data32) async throws {
+        segmentRootByWorkPackageHash.removeValue(forKey: segmentRoot)
+    }
+
+    public func get(erasureRoot: Data32, index: UInt16) async throws -> Data4104? {
+        chunks[erasureRoot]?[index]
+    }
+
+    public func set(data: Data4104, erasureRoot: Data32, index: UInt16) async throws {
+        chunks[erasureRoot, default: [:]][index] = data
+    }
+}
+
+extension InMemoryDataStoreBackend: DataStoreNetworkProtocol {
+    public func fetchRemoteChunk(erasureRoot _: Data32, shardIndex _: UInt16, segmentIndices _: [UInt16]) async throws -> Data12? {
+        nil
+    }
+}

Blockchain/Sources/Blockchain/BlockchainServices.swift

@@ -32,7 +32,8 @@ public class BlockchainServices {
         self.genesisBlock = genesisBlock
         self.genesisState = genesisState
         dataProvider = try! await BlockchainDataProvider(InMemoryDataProvider(genesisState: genesisState, genesisBlock: genesisBlock))
-        dataStore = DataStore(InMemoryDataStore(), basePath: URL(fileURLWithPath: "/tmp/boka-test-data"))
+        let dataStoreBackend = InMemoryDataStoreBackend()
+        dataStore = DataStore(dataStoreBackend, dataStoreBackend)
 
         storeMiddleware = StoreMiddleware()
         eventBus = EventBus(eventMiddleware: .serial(Middleware(storeMiddleware), .noError), handlerMiddleware: .noError)

Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift

@@ -85,8 +85,7 @@ public struct ProtocolConfig: Sendable, Codable, Equatable {
     /// WM = 2^11: The maximum number of entries in a work-package manifest.
     public var maxWorkPackageManifestEntries: Int
 
-    /// WB = 12 * 2^20: The maximum size of an encoded work-package together with its extrinsic data and import impli-
-    /// cations, in octets.
+    /// WB = 12 * 2^20: The maximum size of an encoded work-package together with its extrinsic data and import implications, in octets.
     public var maxEncodedWorkPackageSize: Int
 
     /// WG = WP*WE = 4104: The size of a segment in octets.

Blockchain/Sources/Blockchain/DataStore/DataStore.swift

@@ -5,6 +5,6 @@ import Utils
 public struct WorkPackageBundle: Sendable, Equatable, Codable {
     public var workPackage: WorkPackage
     public var extrinsic: [Data]
-    public var importSegments: [[Data]]
-    public var justifications: [[Data]]
+    public var importSegments: [Data4104]
+    public var justifications: [Data]
 }