fix and debug (#349)

* fix release

* hostcall fix, add some tests from fuzz reports

* fix mod

* fix and more tests

* add more tests

* remove extra files

* fix lint

* add tests

* fix

Author: qiweiii

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
     - run: sudo apt-get update
     - uses: awalsh128/cache-apt-pkgs-action@latest
       with:
-        packages: librocksdb-dev libzstd-dev libbz2-dev liblz4-dev
+        packages: curl build-essential cmake librocksdb-dev libzstd-dev libbz2-dev liblz4-dev
     - uses: aws-actions/configure-aws-credentials@v4
       with:
         aws-region: us-east-2

Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift

@@ -212,8 +212,8 @@ extension Accumulation {
         var newAuthorizationQueue = authorizationQueue
         var overallAccountChanges = AccountChanges()
         var tempPrivilegedServices: PrivilegedServices?
-        var newDelegator = privilegedServices.delegator
-        var newAssigners = privilegedServices.assigners
+        var newDelegator: ServiceIndex?
+        var newAssigners: ConfigFixedSizeArray<ServiceIndex, ProtocolConfig.TotalNumberOfCores>?
 
         for report in workReports {
             for digest in report.digests {
@@ -260,20 +260,19 @@ extension Accumulation {
 
             servicePreimageSet.formUnion(singleOutput.provide)
 
-            switch service {
             // m'
-            case privilegedServices.manager:
+            if service == privilegedServices.manager {
                 tempPrivilegedServices = PrivilegedServices(
                     manager: singleOutput.state.manager,
                     assigners: singleOutput.state.assigners,
                     delegator: singleOutput.state.delegator,
                     alwaysAcc: singleOutput.state.alwaysAcc
                 )
+            }
+
             // i'
-            case privilegedServices.delegator:
+            if service == privilegedServices.delegator {
                 newValidatorQueue = singleOutput.state.validatorQueue
-            default:
-                break
             }
 
             // v'
@@ -288,7 +287,10 @@ extension Accumulation {
 
             // a'
             if let index = tempPrivilegedServices?.assigners.firstIndex(of: service) {
-                newAssigners[index] = singleOutput.state.assigners[index]
+                if newAssigners == nil {
+                    newAssigners = tempPrivilegedServices?.assigners
+                }
+                newAssigners![index] = singleOutput.state.assigners[index]
             }
 
             accountsRef = singleOutput.state.accounts
@@ -308,8 +310,8 @@ extension Accumulation {
                 validatorQueue: newValidatorQueue ?? validatorQueue,
                 authorizationQueue: newAuthorizationQueue,
                 manager: tempPrivilegedServices?.manager ?? privilegedServices.manager,
-                assigners: newAssigners,
-                delegator: newDelegator,
+                assigners: newAssigners ?? tempPrivilegedServices?.assigners ?? privilegedServices.assigners,
+                delegator: newDelegator ?? tempPrivilegedServices?.delegator ?? privilegedServices.delegator,
                 alwaysAcc: tempPrivilegedServices?.alwaysAcc ?? privilegedServices.alwaysAcc,
                 entropy: state.entropy
             ),
@@ -533,15 +535,6 @@ extension Accumulation {
             timeslot: timeslot
         )
 
-        authorizationQueue = accumulateOutput.state.authorizationQueue
-        validatorQueue = accumulateOutput.state.validatorQueue
-        privilegedServices = PrivilegedServices(
-            manager: accumulateOutput.state.manager,
-            assigners: accumulateOutput.state.assigners,
-            delegator: accumulateOutput.state.delegator,
-            alwaysAcc: accumulateOutput.state.alwaysAcc
-        )
-
         // transfers execution + transfers statistics
         var transferGroups = [ServiceIndex: [DeferredTransfers]]()
         var transfersStats = TransfersStats()
@@ -552,7 +545,7 @@ extension Accumulation {
             let gasUsed = try await onTransfer(
                 config: config,
                 serviceIndex: service,
-                serviceAccounts: accountsMutRef,
+                serviceAccounts: accumulateOutput.state.accounts,
                 timeslot: timeslot,
                 entropy: entropy,
                 transfers: transfers
@@ -564,6 +557,16 @@ extension Accumulation {
 
         self = accumulateOutput.state.accounts.value as! Self
 
+        // update non-accounts state after accounts updated
+        authorizationQueue = accumulateOutput.state.authorizationQueue
+        validatorQueue = accumulateOutput.state.validatorQueue
+        privilegedServices = PrivilegedServices(
+            manager: accumulateOutput.state.manager,
+            assigners: accumulateOutput.state.assigners,
+            delegator: accumulateOutput.state.delegator,
+            alwaysAcc: accumulateOutput.state.alwaysAcc
+        )
+
         // update accumulation history
         let accumulated = accumulatableReports[0 ..< accumulateOutput.numAccumulated]
         let newHistoryItem = Set(accumulated.map(\.packageSpecification.workPackageHash))

Blockchain/Sources/Blockchain/State/ServiceAccounts.swift

@@ -65,7 +65,12 @@ public class ServiceAccountsMutRef {
     }
 
     public func addNew(serviceAccount index: ServiceIndex, account: ServiceAccount) async throws {
-        ref.value.set(serviceAccount: index, account: account.toDetails())
+        // set new account details with zero footprint initially
+        var accountDetails = account.toDetails()
+        accountDetails.itemsCount = 0
+        accountDetails.totalByteLength = 0
+        ref.value.set(serviceAccount: index, account: accountDetails)
+
         for (key, value) in account.storage {
             try await ref.value.set(serviceAccount: index, storageKey: key, value: value)
         }

Blockchain/Sources/Blockchain/VMInvocations/Constants.swift

@@ -1 +1 @@
-public let serviceIndexModValue = UInt32((1 << 32) - (1 << 9))
+public let serviceIndexModValue = UInt32(UInt64(1) << 32 - (1 << 9))

Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift

@@ -225,13 +225,16 @@ public class Lookup: HostCall {
         }
 
         let preimageHash = try Data32(state.readMemory(address: regs[0], length: 32))
+        logger.debug("preimageHash: \(String(describing: preimageHash))")
 
         let value: Data? = if let service, let preimageHash {
             try await serviceAccounts.value.get(serviceAccount: service, preimageHash: preimageHash)
         } else {
             nil
         }
 
+        logger.debug("value: \(String(describing: value))")
+
         let reg10: UInt64 = state.readRegister(Registers.Index(raw: 10))
         let reg11: UInt64 = state.readRegister(Registers.Index(raw: 11))
 
@@ -774,7 +777,8 @@ public class Bless: HostCall {
         if state.isMemoryReadable(address: regs[1], length: 4 * config.value.totalNumberOfCores) {
             assigners = try JamDecoder.decode(
                 ConfigFixedSizeArray<ServiceIndex, ProtocolConfig.TotalNumberOfCores>.self,
-                from: state.readMemory(address: regs[1], length: 4 * config.value.totalNumberOfCores)
+                from: state.readMemory(address: regs[1], length: 4 * config.value.totalNumberOfCores),
+                withConfig: config
             )
         }
 
@@ -797,6 +801,11 @@ public class Bless: HostCall {
         } else if ![regs[0], regs[2]].allSatisfy({ $0 >= 0 && $0 <= Int(UInt32.max) }) {
             state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.WHO.rawValue)
         } else {
+            logger.debug("manager: \(regs[0])")
+            logger.debug("assigners: \(String(describing: assigners))")
+            logger.debug("delegator: \(regs[2])")
+            logger.debug("alwaysAcc: \(String(describing: alwaysAcc))")
+
             state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue)
             x.state.manager = regs[0]
             x.state.assigners = assigners!
@@ -919,7 +928,7 @@ public class New: HostCall {
     }
 
     private func bump(i: ServiceIndex) -> ServiceIndex {
-        256 + ((i - 256 + 42) & (serviceIndexModValue - 1))
+        256 + ((i - 256 + 42) % serviceIndexModValue)
     }
 
     public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws {

Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift

@@ -31,9 +31,9 @@ public func accumulate(
         return .init(state: state, transfers: [], commitment: nil, gasUsed: Gas(0), provide: [])
     }
 
-    let initialIndex = try Blake2b256.hash(JamEncoder.encode(UInt(serviceIndex), state.entropy, UInt(timeslot))).data.decode(UInt32.self)
+    let initialIndex = try Blake2b256.hash(JamEncoder.encode(serviceIndex, state.entropy, timeslot)).data.decode(UInt32.self)
     let nextAccountIndex = try await AccumulateContext.check(
-        i: initialIndex % serviceIndexModValue + 256,
+        i: (initialIndex % serviceIndexModValue) + 256,
         accounts: state.accounts.toRef()
     )
 

Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift

@@ -12,21 +12,23 @@ public func onTransfer(
     transfers: [DeferredTransfers]
 ) async throws -> Gas {
     guard var account = try await serviceAccounts.value.get(serviceAccount: serviceIndex),
-          let preimage = try await serviceAccounts.value.get(serviceAccount: serviceIndex, preimageHash: account.codeHash),
           !transfers.isEmpty
     else {
         return Gas(0)
     }
 
-    let codeBlob = try CodeAndMeta(data: preimage).codeBlob
+    account.balance += transfers.reduce(Balance(0)) { $0 + $1.amount }
+    serviceAccounts.set(serviceAccount: serviceIndex, account: account)
 
-    guard codeBlob.count <= config.value.maxServiceCodeSize else {
+    guard let preimage = try await serviceAccounts.value.get(serviceAccount: serviceIndex, preimageHash: account.codeHash) else {
         return Gas(0)
     }
 
-    account.balance += transfers.reduce(Balance(0)) { $0 + $1.amount }
+    let codeBlob = try CodeAndMeta(data: preimage).codeBlob
 
-    serviceAccounts.set(serviceAccount: serviceIndex, account: account)
+    guard codeBlob.count <= config.value.maxServiceCodeSize else {
+        return Gas(0)
+    }
 
     let contextContent = OnTransferContext.ContextType(serviceIndex: serviceIndex, accounts: serviceAccounts)
     let ctx = OnTransferContext(context: contextContent, config: config, entropy: entropy, transfers: transfers)

JAMTests/Package.swift

@@ -35,6 +35,7 @@ let package = Package(
                 .copy("../../jamtestvectors"),
                 .copy("../../jamduna"),
                 .copy("../../javajam"),
+                .copy("../../other"),
             ],
             swiftSettings: [
                 .interoperabilityMode(.Cxx),

JAMTests/Sources/JAMTests/TestLoader.swift

@@ -32,6 +32,7 @@ enum TestsSource: String {
     case jamduna
     case javajam
     case jamixir
+    case other
 }
 
 enum TestLoader {

JAMTests/Tests/JAMTests/jamtestnet/OtherTests.swift

@@ -0,0 +1,56 @@
+import Foundation
+import Testing
+import Utils
+
+@testable import JAMTests
+
+struct OtherTests {
+    // empty array means run all
+    static let testFilters: [(String, String)] = []
+
+    // ignore tests
+    static let ignore: [(String, String)] = [
+        ("0.6.7/1754982630", "00000008"), // NOTE: seems this should fail on .invalidResultCodeHash
+    ]
+
+    static func discoverTests() throws -> [Testcase] {
+        let otherPath = Bundle.module.resourcePath! + "/other"
+        var allTestcases: [Testcase] = []
+
+        for version in try FileManager.default.contentsOfDirectory(atPath: otherPath).sorted() {
+            guard !version.starts(with: ".") else { continue }
+
+            let versionPath = otherPath + "/" + version
+            for timestamp in try FileManager.default.contentsOfDirectory(atPath: versionPath).sorted() {
+                guard !timestamp.starts(with: ".") else { continue }
+
+                let path = "\(version)/\(timestamp)"
+
+                let testcases = try JamTestnet.loadTests(path: path, src: .other)
+
+                let testsExceptIgnore = testcases.filter { testcase in
+                    for (ignorePath, prefix) in ignore where path == ignorePath && testcase.description.starts(with: prefix) {
+                        return false
+                    }
+                    return true
+                }
+
+                if testFilters.isEmpty {
+                    allTestcases.append(contentsOf: testsExceptIgnore)
+                } else {
+                    for (filterPath, filterPrefix) in testFilters where filterPath == path {
+                        let filtered = testsExceptIgnore.filter { $0.description.starts(with: filterPrefix) }
+                        allTestcases.append(contentsOf: filtered)
+                    }
+                }
+            }
+        }
+
+        return allTestcases
+    }
+
+    @Test(arguments: try discoverTests())
+    func otherTests(_ testcase: Testcase) async throws {
+        try await TraceTest.test(testcase)
+    }
+}