Accumulate tests (#301)

* rename and clean up

* clean up vm invocation calls

* refactor

* fix get code blob

* fix edit queue

* tests

* fix lint

Author: qiweiii

Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift

@@ -76,21 +76,3 @@ public struct AccumlateResultContext {
     /// y
     public var yield: Data32?
 }
-
-public protocol AccumulateFunction {
-    func invoke(
-        config: ProtocolConfigRef,
-        // prior accounts
-        accounts: inout some ServiceAccounts,
-        // u
-        state: AccumulateState,
-        // s
-        serviceIndex: ServiceIndex,
-        // g
-        gas: Gas,
-        // o
-        arguments: [AccumulateArguments],
-        initialIndex: ServiceIndex,
-        timeslot: TimeslotIndex
-    ) async throws -> (state: AccumulateState, transfers: [DeferredTransfers], result: Data32?, gas: Gas)
-}

Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift

@@ -58,21 +58,19 @@ public struct SingleAccumulationOutput {
 
 public protocol Accumulation: ServiceAccounts {
     var timeslot: TimeslotIndex { get }
-    var privilegedServices: PrivilegedServices { get }
+    var privilegedServices: PrivilegedServices { get set }
     var validatorQueue: ConfigFixedSizeArray<
         ValidatorKey, ProtocolConfig.TotalNumberOfValidators
-    > { get }
+    > { get set }
     var authorizationQueue: ConfigFixedSizeArray<
         ConfigFixedSizeArray<
             Data32,
             ProtocolConfig.MaxAuthorizationsQueueItems
         >,
         ProtocolConfig.TotalNumberOfCores
-    > { get }
-    var accumlateFunction: AccumulateFunction { get }
-    var onTransferFunction: OnTransferFunction { get }
-    var accumulationQueue: StateKeys.AccumulationQueueKey.Value { get }
-    var accumulationHistory: StateKeys.AccumulationHistoryKey.Value { get }
+    > { get set }
+    var accumulationQueue: StateKeys.AccumulationQueueKey.Value { get set }
+    var accumulationHistory: StateKeys.AccumulationHistoryKey.Value { get set }
 }
 
 extension Accumulation {
@@ -105,7 +103,7 @@ extension Accumulation {
             }
         }
 
-        let (newState, transfers, commitment, gasUsed) = try await accumlateFunction.invoke(
+        let (newState, transfers, commitment, gasUsed) = try await accumulate(
             config: config,
             accounts: &self,
             state: state,
@@ -269,30 +267,29 @@ extension Accumulation {
         }
     }
 
-    // E: edit the accumulation queue, remove the dependencies of the items that are already accumulated
-    public func editAccumulatedItems(items: inout [AccumulationQueueItem], accumulatedPackages: Set<Data32>) {
-        for index in items.indices
-            where !accumulatedPackages.contains(items[index].workReport.packageSpecification.workPackageHash)
-        {
-            items[index].dependencies.subtract(accumulatedPackages)
+    // E: edit the accumulation queue items when some work reports are accumulated
+    private func editQueue(items: inout [AccumulationQueueItem], accumulatedPackages: Set<Data32>) {
+        items = items.filter { !accumulatedPackages.contains($0.workReport.packageSpecification.workPackageHash) }
+
+        for i in items.indices {
+            items[i].dependencies.subtract(accumulatedPackages)
         }
     }
 
-    // Q: find the reports that have no dependencies
-    private func findNoDepsReports(items: inout [AccumulationQueueItem]) -> [WorkReport] {
+    // Q: provides the sequence of work-reports which are accumulatable given queue items
+    private func getAccumulatables(items: inout [AccumulationQueueItem]) -> [WorkReport] {
         let noDepsReports = items.filter(\.dependencies.isEmpty).map(\.workReport)
         if noDepsReports.isEmpty {
             return []
         } else {
-            editAccumulatedItems(items: &items, accumulatedPackages: Set(noDepsReports.map(\.packageSpecification.workPackageHash)))
-            return noDepsReports + findNoDepsReports(items: &items)
+            editQueue(items: &items, accumulatedPackages: Set(noDepsReports.map(\.packageSpecification.workPackageHash)))
+            return noDepsReports + getAccumulatables(items: &items)
         }
     }
 
     // newly available work-reports, W, are partitioned into two sequences based on the condition of having zero prerequisite work-reports
-    public func partitionWorkReports(
-        availableReports: [WorkReport],
-        history: StateKeys.AccumulationHistoryKey.Value
+    private func partitionWorkReports(
+        availableReports: [WorkReport]
     ) -> (zeroPrereqReports: [WorkReport], newQueueItems: [AccumulationQueueItem]) {
         let zeroPrereqReports = availableReports.filter { report in
             report.refinementContext.prerequisiteWorkPackages.isEmpty && report.lookup.isEmpty
@@ -308,40 +305,42 @@ extension Accumulation {
             ))
         }
 
-        editAccumulatedItems(
+        editQueue(
             items: &newQueueItems,
-            accumulatedPackages: Set(history.array.reduce(into: Set<Data32>()) { $0.formUnion($1.array) })
+            accumulatedPackages: Set(accumulationHistory.array.reduce(into: Set<Data32>()) { $0.formUnion($1.array) })
         )
 
         return (zeroPrereqReports, newQueueItems)
     }
 
-    public func getAccumulatableReports(
-        index: Int, availableReports: [WorkReport],
-        history: StateKeys.AccumulationHistoryKey.Value
+    // get all the work reports that can be accumulated in this block
+    private func getAllAccumulatableReports(
+        availableReports: [WorkReport],
+        index: Int
     ) -> (accumulatableReports: [WorkReport], newQueueItems: [AccumulationQueueItem]) {
-        let (zeroPrereqReports, newQueueItems) = partitionWorkReports(availableReports: availableReports, history: history)
+        let (zeroPrereqReports, newQueueItems) = partitionWorkReports(availableReports: availableReports)
 
         let rightQueueItems = accumulationQueue.array[index...]
         let leftQueueItems = accumulationQueue.array[0 ..< index]
         var allQueueItems = rightQueueItems.flatMap(\.self) + leftQueueItems.flatMap(\.self) + newQueueItems
 
-        editAccumulatedItems(items: &allQueueItems, accumulatedPackages: Set(zeroPrereqReports.map(\.packageSpecification.workPackageHash)))
+        editQueue(items: &allQueueItems, accumulatedPackages: Set(zeroPrereqReports.map(\.packageSpecification.workPackageHash)))
 
-        return (zeroPrereqReports + findNoDepsReports(items: &allQueueItems), newQueueItems)
+        return (zeroPrereqReports + getAccumulatables(items: &allQueueItems), newQueueItems)
     }
 
-    public mutating func update(
+    // accumulate execution
+    private mutating func execution(
         config: ProtocolConfigRef,
         workReports: [WorkReport],
         entropy: Data32,
         timeslot: TimeslotIndex
-    ) async throws -> (numAccumulated: Int, state: AccumulateState, commitments: Set<Commitment>) {
+    ) async throws -> AccumulationOutput {
         let sumPrevilegedGas = privilegedServices.basicGas.values.reduce(Gas(0)) { $0 + $1.value }
         let minTotalGas = config.value.workReportAccumulationGas * Gas(config.value.totalNumberOfCores) + sumPrevilegedGas
         let gasLimit = max(config.value.totalAccumulationGas, minTotalGas)
 
-        let res = try await outerAccumulate(
+        return try await outerAccumulate(
             config: config,
             state: AccumulateState(
                 newServiceAccounts: [:],
@@ -355,23 +354,93 @@ extension Accumulation {
             entropy: entropy,
             timeslot: timeslot
         )
+    }
 
-        var transferGroups = [ServiceIndex: [DeferredTransfers]]()
+    /// Accumulate execution, state integration and deferred transfers
+    ///
+    /// Return accumulation-result merkle tree root
+    public mutating func update(
+        config: ProtocolConfigRef,
+        availableReports: [WorkReport],
+        timeslot: TimeslotIndex,
+        prevTimeslot: TimeslotIndex,
+        entropy: Data32
+    ) async throws -> Data32 {
+        let index = Int(timeslot) %% config.value.epochLength
+
+        var (accumulatableReports, newQueueItems) = getAllAccumulatableReports(
+            availableReports: availableReports,
+            index: index
+        )
 
-        for transfer in res.transfers {
-            transferGroups[transfer.destination, default: []].append(transfer)
+        let accumulateOutput = try await execution(
+            config: config,
+            workReports: accumulatableReports,
+            entropy: entropy,
+            timeslot: timeslot
+        )
+
+        authorizationQueue = accumulateOutput.state.authorizationQueue
+        validatorQueue = accumulateOutput.state.validatorQueue
+        privilegedServices = accumulateOutput.state.privilegedServices
+
+        // add new service accounts
+        for (service, account) in accumulateOutput.state.newServiceAccounts {
+            set(serviceAccount: service, account: account.toDetails())
+            for (hash, value) in account.storage {
+                set(serviceAccount: service, storageKey: hash, value: value)
+            }
+            for (hash, value) in account.preimages {
+                set(serviceAccount: service, preimageHash: hash, value: value)
+            }
+            for (hashLength, value) in account.preimageInfos {
+                set(serviceAccount: service, preimageHash: hashLength.hash, length: hashLength.length, value: value)
+            }
         }
 
+        // transfers
+        var transferGroups = [ServiceIndex: [DeferredTransfers]]()
+        for transfer in accumulateOutput.transfers {
+            transferGroups[transfer.destination, default: []].append(transfer)
+        }
         for (service, transfers) in transferGroups {
-            try await onTransferFunction.invoke(
+            try await onTransfer(
                 config: config,
-                service: service,
+                serviceIndex: service,
                 serviceAccounts: &self,
                 timeslot: timeslot,
                 transfers: transfers
             )
         }
 
-        return (res.numAccumulated, res.state, res.commitments)
+        // update accumulation history
+        let accumulated = accumulatableReports[0 ..< accumulateOutput.numAccumulated]
+        let newHistoryItem = Set(accumulated.map(\.packageSpecification.workPackageHash))
+        for i in 0 ..< config.value.epochLength {
+            if i == config.value.epochLength - 1 {
+                accumulationHistory[i] = .init(newHistoryItem)
+            } else {
+                accumulationHistory[i] = accumulationHistory[i + 1]
+            }
+        }
+
+        // update accumulation queue
+        for i in 0 ..< config.value.epochLength {
+            let queueIdx = (index - i) %% config.value.epochLength
+            if i == 0 {
+                editQueue(items: &newQueueItems, accumulatedPackages: newHistoryItem)
+                accumulationQueue[queueIdx] = newQueueItems
+            } else if i >= 1, i < timeslot - prevTimeslot {
+                accumulationQueue[queueIdx] = []
+            } else {
+                editQueue(items: &accumulationQueue[queueIdx], accumulatedPackages: newHistoryItem)
+            }
+        }
+
+        // accumulate root
+        let nodes = try accumulateOutput.commitments.map { try JamEncoder.encode($0.serviceIndex) + JamEncoder.encode($0.hash) }
+        let root = Merklization.binaryMerklize(nodes, hasher: Keccak.self)
+
+        return root
     }
 }

Blockchain/Sources/Blockchain/RuntimeProtocols/OnTransferFunction.swift

@@ -170,13 +170,13 @@ public final class Runtime {
             // depends on Safrole and Disputes
             let availableReports = try updateReports(block: block, state: &newState)
 
-            // accumulation
-            try await accumulate(
+            // accumulate
+            let accumulateRoot = try await newState.update(
                 config: config,
-                block: block,
                 availableReports: availableReports,
-                state: &newState,
-                prevTimeslot: prevState.value.timeslot
+                timeslot: block.header.timeslot,
+                prevTimeslot: prevState.value.timeslot,
+                entropy: newState.entropyPool.t0
             )
 
             do {
@@ -200,7 +200,7 @@ public final class Runtime {
             )
 
             // after reports as it need old recent history
-            try updateRecentHistory(block: block, state: &newState)
+            try updateRecentHistory(block: block, state: &newState, accumulateRoot: accumulateRoot)
 
             try await newState.save()
         } catch let error as Error {
@@ -216,79 +216,15 @@ public final class Runtime {
         return StateRef(newState)
     }
 
-    // accumulation related state updates
-    public func accumulate(
-        config: ProtocolConfigRef,
-        block: BlockRef,
-        availableReports: [WorkReport],
-        state: inout State,
-        prevTimeslot: TimeslotIndex
-    ) async throws {
-        let curIndex = Int(block.header.timeslot) % config.value.epochLength
-        var (accumulatableReports, newQueueItems) = state.getAccumulatableReports(
-            index: curIndex,
-            availableReports: availableReports,
-            history: state.accumulationHistory
-        )
-
-        // accumulate and transfers
-        let (numAccumulated, accumulateState, _) = try await state.update(
-            config: config,
-            workReports: accumulatableReports,
-            entropy: state.entropyPool.t0,
-            timeslot: block.header.timeslot
-        )
-
-        state.authorizationQueue = accumulateState.authorizationQueue
-        state.validatorQueue = accumulateState.validatorQueue
-        state.privilegedServices = accumulateState.privilegedServices
-        for (service, account) in accumulateState.newServiceAccounts {
-            state[serviceAccount: service] = account.toDetails()
-            for (hash, value) in account.storage {
-                state[serviceAccount: service, storageKey: hash] = value
-            }
-            for (hash, value) in account.preimages {
-                state[serviceAccount: service, preimageHash: hash] = value
-            }
-            for (hashLength, value) in account.preimageInfos {
-                state[serviceAccount: service, preimageHash: hashLength.hash, length: hashLength.length] = value
-            }
-        }
-
-        // update accumulation history
-        let accumulated = accumulatableReports[0 ..< numAccumulated]
-        let newHistoryItem = Set(accumulated.map(\.packageSpecification.workPackageHash))
-        for i in 0 ..< config.value.epochLength {
-            if i == config.value.epochLength - 1 {
-                state.accumulationHistory[i] = .init(newHistoryItem)
-            } else {
-                state.accumulationHistory[i] = state.accumulationHistory[i + 1]
-            }
-        }
-
-        // update accumulation queue
-        for i in 0 ..< config.value.epochLength {
-            let queueIdx = (curIndex - i) %% config.value.epochLength
-            if i == 0 {
-                state.editAccumulatedItems(items: &newQueueItems, accumulatedPackages: newHistoryItem)
-                state.accumulationQueue[queueIdx] = newQueueItems
-            } else if i >= 1, i < state.timeslot - prevTimeslot {
-                state.accumulationQueue[queueIdx] = []
-            } else {
-                state.editAccumulatedItems(items: &state.accumulationQueue[queueIdx], accumulatedPackages: newHistoryItem)
-            }
-        }
-    }
-
-    public func updateRecentHistory(block: BlockRef, state newState: inout State) throws {
+    public func updateRecentHistory(block: BlockRef, state newState: inout State, accumulateRoot: Data32) throws {
         let lookup: [Data32: Data32] = Dictionary(uniqueKeysWithValues: block.extrinsic.reports.guarantees.map { (
             $0.workReport.packageSpecification.workPackageHash,
             $0.workReport.packageSpecification.segmentRoot
         ) })
         newState.recentHistory.update(
             headerHash: block.hash,
             parentStateRoot: block.header.priorStateRoot,
-            accumulateRoot: Data32(), // TODO: calculate accumulation result
+            accumulateRoot: accumulateRoot,
             lookup: lookup
         )
     }