Added lightnet and created integration tests

Author: rpanic

packages/deployment/docker/docker-compose.yml

@@ -7,3 +7,5 @@ include:
   - ./sequencer/docker-compose.yml
   # Enabled by worker profile
   - ./worker/docker-compose.yml
+
+  - ./lightnet/docker-compose.yml

packages/deployment/docker/lightnet/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  lightnet:
+    image: o1labs/mina-local-network:compatible-latest-lightnet
+    container_name: lightnet
+    environment:
+      - RUN_ARCHIVE_NODE=true
+      - LOG_LEVEL=INFO
+      - PROOF_LEVEL=none
+      - NETWORK_TYPE=single-node
+    ports:
+      - 3085:3085
+      - 8080:8080
+      - 8181:8181
+#     archive endpoints
+      - 5432:5432
+      - 8282:8282

packages/sequencer/package.json

@@ -11,6 +11,7 @@
     "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js",
     "test": "npm run test:file -- ./test/**",
     "test:watch": "npm run test:file -- ./test/** --watch",
+    "integration": "npm run test:file -- ./test-integration/** --runInBand",
     "start": "npm run build && node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads --es-module-specifier-resolution=node ./dist/src/entry.js"
   },
   "main": "dist/index.js",

packages/sequencer/src/protocol/baselayer/accounts/MinaBlockchainAccounts.ts

@@ -0,0 +1,90 @@
+import { injectable, inject } from "tsyringe";
+import { range } from "@proto-kit/common";
+import { BaseLayer } from "../BaseLayer";
+import { MinaBaseLayer } from "../MinaBaseLayer";
+import { PrivateKey, Mina, Lightnet, PublicKey, AccountUpdate } from "o1js";
+import { MinaTransactionSender } from "../../../settlement/transactions/MinaTransactionSender";
+import { FeeStrategy } from "../fees/FeeStrategy";
+
+type LocalBlockchain = Awaited<ReturnType<typeof Mina.LocalBlockchain>>;
+
+@injectable()
+export class MinaBlockchainAccounts {
+  public constructor(
+    @inject("BaseLayer")
+    private readonly baseLayer: BaseLayer,
+    @inject("TransactionSender")
+    private readonly transactionSender: MinaTransactionSender,
+    @inject("FeeStrategy")
+    private readonly feeStrategy: FeeStrategy
+  ) {}
+
+  private keysRetrieved = 0;
+
+  private isMinaBaseLayer(
+    baseLayer: BaseLayer | undefined
+  ): baseLayer is MinaBaseLayer {
+    return baseLayer !== undefined && baseLayer instanceof MinaBaseLayer;
+  }
+
+  public async getFundedAccounts(num: number = 1): Promise<PrivateKey[]> {
+    const { baseLayer } = this;
+    if (!this.isMinaBaseLayer(baseLayer)) {
+      throw new Error("Baselayer not defined or not subclass of MinaBaseLayer");
+    }
+    if (baseLayer.config.network.type === "local") {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      const local = baseLayer.network! as LocalBlockchain;
+      const accounts = local.testAccounts.slice(
+        this.keysRetrieved,
+        this.keysRetrieved + num
+      );
+      this.keysRetrieved += num;
+      return accounts.map((acc) => acc.key);
+    }
+    if (baseLayer.config.network.type === "lightnet") {
+      return await Promise.all(
+        range(num).map(async (i) => {
+          const pair = await Lightnet.acquireKeyPair({
+            isRegularAccount: true,
+          });
+          return pair.privateKey;
+        })
+      );
+    }
+    throw new Error("Can't acquire keys for remote non-lighnet network");
+  }
+
+  public async fundAccountFrom(
+    sender: PrivateKey,
+    receiver: PublicKey,
+    amount: number
+  ) {
+    const { baseLayer } = this;
+    if (!this.isMinaBaseLayer(baseLayer)) {
+      throw new Error("Baselayer not defined or not subclass of MinaBaseLayer");
+    }
+    if (baseLayer.config.network.type === "local") {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      (baseLayer.network as LocalBlockchain).addAccount(
+        receiver,
+        amount.toString()
+      );
+    } else {
+      const tx = await Mina.transaction(
+        { sender: sender.toPublicKey(), fee: this.feeStrategy.getFee() },
+        async () => {
+          AccountUpdate.fundNewAccount(sender.toPublicKey());
+          AccountUpdate.createSigned(sender.toPublicKey()).send({
+            to: receiver,
+            amount,
+          });
+        }
+      );
+      await this.transactionSender.proveAndSendTransaction(
+        tx.sign([sender]),
+        "included"
+      );
+    }
+  }
+}

packages/sequencer/src/settlement/SettlementModule.ts

@@ -11,6 +11,7 @@ import {
   SettlementContractConfig,
   MandatorySettlementModulesRecord,
   MandatoryProtocolModulesRecord,
+  BlockProverPublicOutput,
 } from "@proto-kit/protocol";
 import {
   AccountUpdate,
@@ -20,6 +21,7 @@ import {
   PublicKey,
   Signature,
   Transaction,
+  fetchAccount,
 } from "o1js";
 import { inject } from "tsyringe";
 import {
@@ -45,6 +47,7 @@ import { AsyncMerkleTreeStore } from "../state/async/AsyncMerkleTreeStore";
 import { CachedMerkleTreeStore } from "../state/merkle/CachedMerkleTreeStore";
 import { BlockProofSerializer } from "../protocol/production/helpers/BlockProofSerializer";
 import { Settlement } from "../storage/model/Settlement";
+import { FeeStrategy } from "../protocol/baselayer/fees/FeeStrategy";
 
 import { IncomingMessageAdapter } from "./messages/IncomingMessageAdapter";
 import type { OutgoingMessageQueue } from "./messages/WithdrawalQueue";
@@ -107,7 +110,9 @@ export class SettlementModule
     @inject("TransactionSender")
     private readonly transactionSender: MinaTransactionSender,
     @inject("AreProofsEnabled")
-    private readonly areProofsEnabled: AreProofsEnabled
+    private readonly areProofsEnabled: AreProofsEnabled,
+    @inject("FeeStrategy")
+    private readonly feeStrategy: FeeStrategy
   ) {
     super();
   }
@@ -234,8 +239,8 @@ export class SettlementModule
           sender: feepayer.toPublicKey(),
           // eslint-disable-next-line no-plusplus
           nonce: nonce++,
-          fee: String(0.01 * 1e9),
-          memo: "Protokit settle",
+          fee: this.feeStrategy.getFee(),
+          memo: "roll up actions",
         },
         async () => {
           await settlement.rollupOutgoingMessages(
@@ -246,7 +251,10 @@ export class SettlementModule
 
       const signedTx = this.signTransaction(tx, [feepayer]);
 
-      await this.transactionSender.proveAndSendTransaction(signedTx);
+      await this.transactionSender.proveAndSendTransaction(
+        signedTx,
+        "included"
+      );
 
       this.outgoingMessageQueue.pop(OUTGOING_MESSAGE_BATCH_SIZE);
 
@@ -259,33 +267,51 @@ export class SettlementModule
   }
   /* eslint-enable no-await-in-loop */
 
+  private async fetchContractAccounts() {
+    const contracts = this.getContracts();
+    if (contracts !== undefined) {
+      const c1 = await fetchAccount({
+        publicKey: contracts.settlement.address,
+        tokenId: contracts.settlement.tokenId,
+      });
+      const c2 = await fetchAccount({
+        publicKey: contracts.dispatch.address,
+        tokenId: contracts.dispatch.tokenId,
+      });
+    }
+  }
+
   public async settleBatch(
     batch: SettleableBatch,
     options: {
       nonce?: number;
     } = {}
   ): Promise<Settlement> {
+    await this.fetchContractAccounts();
     const { settlement, dispatch } = this.getContracts();
     const { feepayer } = this.config;
 
     log.debug("Preparing settlement");
 
-    const lastSettlementL1Block = settlement.lastSettlementL1Block.get().value;
+    const lastSettlementL1BlockHeight =
+      settlement.lastSettlementL1BlockHeight.get().value;
     const signature = Signature.create(feepayer, [
       BATCH_SIGNATURE_PREFIX,
-      lastSettlementL1Block,
+      lastSettlementL1BlockHeight,
     ]);
 
-    const fromSequenceStateHash = dispatch.honoredMessagesHash.get();
-    const latestSequenceStateHash = settlement.account.actionState.get();
+    const fromSequenceStateHash = BlockProverPublicOutput.fromFields(
+      batch.proof.publicOutput.map((x) => Field(x))
+    ).incomingMessagesHash;
+    const latestSequenceStateHash = dispatch.account.actionState.get();
 
     // Fetch actions and store them into the messageStorage
     const actions = await this.incomingMessagesAdapter.getPendingMessages(
-      settlement.address,
+      dispatch.address,
       {
         fromActionHash: fromSequenceStateHash.toString(),
         toActionHash: latestSequenceStateHash.toString(),
-        fromL1Block: Number(lastSettlementL1Block.toString()),
+        fromL1BlockHeight: Number(lastSettlementL1BlockHeight.toString()),
       }
     );
     await this.messageStorage.pushMessages(
@@ -302,7 +328,7 @@ export class SettlementModule
       {
         sender: feepayer.toPublicKey(),
         nonce: options?.nonce,
-        fee: String(0.01 * 1e9),
+        fee: this.feeStrategy.getFee(),
         memo: "Protokit settle",
       },
       async () => {
@@ -320,7 +346,7 @@ export class SettlementModule
 
     this.signTransaction(tx, [feepayer]);
 
-    await this.transactionSender.proveAndSendTransaction(tx);
+    await this.transactionSender.proveAndSendTransaction(tx, "included");
 
     log.info("Settlement transaction send queued");
 
@@ -355,16 +381,24 @@ export class SettlementModule
       ? new SignedSettlementPermissions()
       : new ProvenSettlementPermissions();
 
+    if (this.baseLayer.config.network.type !== "local") {
+      // const ac1 = await fetchAccount({ publicKey: feepayer });
+      // console.log(ac1);
+    }
+
+    // this.baseLayer
+    //       .originalNetwork!
     const tx = await Mina.transaction(
       {
         sender: feepayer,
         nonce,
-        fee: String(0.01 * 1e9),
+        fee: this.feeStrategy.getFee(),
         memo: "Protokit settlement deploy",
       },
       async () => {
         AccountUpdate.fundNewAccount(feepayer, 2);
         await settlement.deploy({
+          // TODO Create compilation task that generates those artifacts if proofs enabled
           verificationKey: undefined,
         });
         settlement.account.permissions.set(permissions.settlementContract());
@@ -379,7 +413,7 @@ export class SettlementModule
 
     // This should already apply the tx result to the
     // cached accounts / local blockchain
-    await this.transactionSender.proveAndSendTransaction(tx);
+    await this.transactionSender.proveAndSendTransaction(tx, "included");
 
     this.addresses = {
       settlement: settlementKey.toPublicKey(),
@@ -394,7 +428,7 @@ export class SettlementModule
       {
         sender: feepayer,
         nonce: nonce + 1,
-        fee: String(0.01 * 1e9),
+        fee: this.feeStrategy.getFee(),
         memo: "Protokit settlement init",
       },
       async () => {
@@ -407,7 +441,10 @@ export class SettlementModule
 
     const initTxSigned = this.signTransaction(initTx, [feepayerKey]);
 
-    await this.transactionSender.proveAndSendTransaction(initTxSigned);
+    await this.transactionSender.proveAndSendTransaction(
+      initTxSigned,
+      "included"
+    );
   }
 
   public async start(): Promise<void> {

packages/sequencer/src/settlement/messages/IncomingMessageAdapter.ts

@@ -14,7 +14,7 @@ export interface IncomingMessageAdapter {
     params: {
       fromActionHash: string;
       toActionHash?: string;
-      fromL1Block: number;
+      fromL1BlockHeight: number;
     }
   ) => Promise<{
     from: string;

packages/sequencer/src/settlement/messages/MinaIncomingMessageAdapter.ts

@@ -73,7 +73,7 @@ export class MinaIncomingMessageAdapter implements IncomingMessageAdapter {
     params: {
       fromActionHash: string;
       toActionHash?: string;
-      fromL1Block: number;
+      fromL1BlockHeight: number;
     }
   ): Promise<{
     from: string;
@@ -96,16 +96,20 @@ export class MinaIncomingMessageAdapter implements IncomingMessageAdapter {
       //   : undefined,
     });
 
-    const events = await network.fetchEvents(address, undefined, {
-      from: UInt32.from(Math.max(params.fromL1Block - 5, 0)),
-    });
-
     if ("error" in actions) {
       throw new Error(
         `Error ${actions.error.statusCode}: ${actions.error.statusText}`
       );
     }
 
+    if (actions.length > 0) {
+      console.log(params.fromL1BlockHeight);
+    }
+
+    const events = await network.fetchEvents(address, undefined, {
+      from: UInt32.from(Math.max(params.fromL1BlockHeight - 5, 0)),
+    });
+
     const messages = await mapSequential(actions, async (action) => {
       // Find events corresponding to the transaction to get the raw args
       const tx = RuntimeTransaction.fromHashData(