Merge branch 'develop' into feature/enable-proving

# Conflicts:
#	packages/protocol/src/prover/block/BlockProver.ts
#	packages/protocol/src/prover/statetransition/StateTransitionProver.ts
#	packages/sequencer/src/protocol/production/BlockTaskFlowService.ts
#	packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts

Author: rpanic

packages/common/src/utils.ts

@@ -164,3 +164,5 @@ type NonMethodKeys<Type> = {
   [Key in keyof Type]: Type[Key] extends Function ? never : Key;
 }[keyof Type];
 export type NonMethods<Type> = Pick<Type, NonMethodKeys<Type>>;
+
+export const MAX_FIELD = Field(Field.ORDER - 1n);

packages/library/src/hooks/RuntimeFeeAnalyzerService.ts

@@ -81,7 +81,7 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule<RuntimeFeeAnal
     });
 
     container.resolve(RuntimeMethodExecutionContext).clear();
-
+    let methodCounter = 0;
     const [values, indexes] =
       await this.runtime.zkProgrammable.zkProgram.reduce<
         Promise<[FeeTreeValues, FeeIndexes]>
@@ -93,7 +93,7 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule<RuntimeFeeAnal
             [FeeTreeValues, FeeIndexes]
           >(
             // eslint-disable-next-line @typescript-eslint/no-shadow
-            ([values, indexes], combinedMethodName, index) => {
+            ([values, indexes], combinedMethodName) => {
               const { rows } = analyzedMethods[combinedMethodName];
               // const rows = 1000;
               const [moduleName, methodName] = combinedMethodName.split(".");
@@ -128,7 +128,8 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule<RuntimeFeeAnal
                 },
                 {
                   ...indexes,
-                  [methodId.toString()]: BigInt(index),
+                  // eslint-disable-next-line no-plusplus
+                  [methodId.toString()]: BigInt(methodCounter++),
                 },
               ];
             },

packages/protocol/src/index.ts

@@ -24,8 +24,6 @@ export * from "./prover/block/accummulators/BlockHashMerkleTree";
 export * from "./prover/block/services/RuntimeVerificationKeyRootService";
 export * from "./prover/statetransition/StateTransitionProver";
 export * from "./prover/statetransition/StateTransitionProvable";
-export * from "./prover/statetransition/StateTransitionWitnessProvider";
-export * from "./prover/statetransition/StateTransitionWitnessProviderReference";
 export * from "./protocol/Protocol";
 export * from "./protocol/ProtocolModule";
 export * from "./protocol/ProtocolEnvironment";

packages/protocol/src/model/StateTransitionProvableBatch.ts

@@ -1,5 +1,10 @@
 import { Bool, Provable, Struct } from "o1js";
-import { range } from "@proto-kit/common";
+import {
+  InMemoryMerkleTreeStorage,
+  range,
+  RollupMerkleTree,
+  RollupMerkleTreeWitness,
+} from "@proto-kit/common";
 
 import { constants } from "../Constants";
 
@@ -60,16 +65,22 @@ export class StateTransitionProvableBatch extends Struct({
     ProvableStateTransitionType,
     constants.stateTransitionProverBatchSize
   ),
+
+  merkleWitnesses: Provable.Array(
+    RollupMerkleTreeWitness,
+    constants.stateTransitionProverBatchSize
+  ),
 }) {
   public static fromMappings(
     transitions: {
       transition: ProvableStateTransition;
       type: ProvableStateTransitionType;
-    }[]
+    }[],
+    merkleWitnesses: RollupMerkleTreeWitness[]
   ): StateTransitionProvableBatch {
     const batch = transitions.map((entry) => entry.transition);
     const transitionTypes = transitions.map((entry) => entry.type);
-
+    const witnesses = merkleWitnesses.slice();
     // Check that order is correct
     let normalSTsStarted = false;
     transitionTypes.forEach((x) => {
@@ -84,16 +95,23 @@ export class StateTransitionProvableBatch extends Struct({
     while (batch.length < constants.stateTransitionProverBatchSize) {
       batch.push(ProvableStateTransition.dummy());
       transitionTypes.push(ProvableStateTransitionType.normal);
+      witnesses.push(
+        new RollupMerkleTree(new InMemoryMerkleTreeStorage()).getWitness(
+          BigInt(0)
+        )
+      );
     }
     return new StateTransitionProvableBatch({
       batch,
       transitionTypes,
+      merkleWitnesses: witnesses,
     });
   }
 
   public static fromTransitions(
     transitions: ProvableStateTransition[],
-    protocolTransitions: ProvableStateTransition[]
+    protocolTransitions: ProvableStateTransition[],
+    merkleWitnesses: RollupMerkleTreeWitness[]
   ): StateTransitionProvableBatch {
     const array = transitions.slice().concat(protocolTransitions);
 
@@ -113,12 +131,14 @@ export class StateTransitionProvableBatch extends Struct({
     return new StateTransitionProvableBatch({
       batch: array,
       transitionTypes,
+      merkleWitnesses,
     });
   }
 
   private constructor(object: {
     batch: ProvableStateTransition[];
     transitionTypes: ProvableStateTransitionType[];
+    merkleWitnesses: RollupMerkleTreeWitness[];
   }) {
     super(object);
   }

packages/protocol/src/prover/block/BlockProvable.ts

@@ -24,6 +24,7 @@ export class BlockProverPublicInput extends Struct({
   blockHashRoot: Field,
   eternalTransactionsHash: Field,
   incomingMessagesHash: Field,
+  blockNumber: Field,
 }) {}
 
 export class BlockProverPublicOutput extends Struct({
@@ -36,15 +37,10 @@ export class BlockProverPublicOutput extends Struct({
   closed: Bool,
   blockNumber: Field,
 }) {
-  public equals(
-    input: BlockProverPublicInput,
-    closed: Bool,
-    blockNumber: Field
-  ): Bool {
+  public equals(input: BlockProverPublicInput, closed: Bool): Bool {
     const output2 = BlockProverPublicOutput.toFields({
       ...input,
       closed,
-      blockNumber,
     });
     const output1 = BlockProverPublicOutput.toFields(this);
     return output1

packages/protocol/src/prover/block/BlockProver.ts

@@ -12,6 +12,7 @@ import {
 import { container, inject, injectable, injectAll } from "tsyringe";
 import {
   AreProofsEnabled,
+  MAX_FIELD,
   PlainZkProgram,
   provableMethod,
   WithZkProgrammable,
@@ -124,10 +125,6 @@ export interface BlockProverState {
   incomingMessagesHash: Field;
 }
 
-function maxField() {
-  return Field(Field.ORDER - 1n);
-}
-
 export type BlockProof = Proof<BlockProverPublicInput, BlockProverPublicOutput>;
 export type RuntimeProof = Proof<void, MethodPublicOutput>;
 
@@ -416,6 +413,11 @@ export class BlockProverProgrammable extends ZkProgrammable<
       "ExecutionData Networkstate doesn't equal public input hash"
     );
 
+    publicInput.blockNumber.assertEquals(
+      MAX_FIELD,
+      "blockNumber has to be MAX for transaction proofs"
+    );
+
     // Verify the [methodId, vk] tuple against the baked-in vk tree root
     const { verificationKey, witness: verificationKeyTreeWitness } =
       verificationKeyWitness;
@@ -445,7 +447,7 @@ export class BlockProverProgrammable extends ZkProgrammable<
 
     return new BlockProverPublicOutput({
       ...stateTo,
-      blockNumber: maxField(),
+      blockNumber: publicInput.blockNumber,
       closed: Bool(false),
     });
   }
@@ -528,14 +530,14 @@ export class BlockProverProgrammable extends ZkProgrammable<
     // stateTransitionProof.verifyIf(Bool(false));
     // stateTransitionProof.verifyIf(stsEmitted);
 
-    // Verify Transaction proof if it has at least 1 tx
+    // Verify Transaction proof if it has at least 1 tx - i.e. the
+    // input and output doesn't match fully
     // We have to compare the whole input and output because we can make no
     // assumptions about the values, since it can be an arbitrary dummy-proof
     const txProofOutput = transactionProof.publicOutput;
     const isEmptyTransition = txProofOutput.equals(
       transactionProof.publicInput,
-      txProofOutput.closed,
-      txProofOutput.blockNumber
+      txProofOutput.closed
     );
     Provable.log("VerifyIf 2", isEmptyTransition.not());
     transactionProof.verifyIf(isEmptyTransition.not());
@@ -620,6 +622,8 @@ export class BlockProverProgrammable extends ZkProgrammable<
     // Calculate the new block index
     const blockIndex = blockWitness.calculateIndex();
 
+    blockIndex.assertEquals(publicInput.blockNumber);
+
     blockWitness
       .calculateRoot(Field(0))
       .assertEquals(
@@ -637,7 +641,7 @@ export class BlockProverProgrammable extends ZkProgrammable<
 
     return new BlockProverPublicOutput({
       ...state,
-      blockNumber: blockIndex,
+      blockNumber: blockIndex.add(1),
       closed: Bool(true),
     });
   }
@@ -744,19 +748,25 @@ export class BlockProverProgrammable extends ZkProgrammable<
     //   assert proof1.height == proof2.height
     // }
 
-    const proof1Height = proof1.publicOutput.blockNumber;
     const proof1Closed = proof1.publicOutput.closed;
-    const proof2Height = proof2.publicOutput.blockNumber;
     const proof2Closed = proof2.publicOutput.closed;
 
-    const isValidTransactionMerge = proof1Height
-      .equals(maxField())
-      .and(proof2Height.equals(proof1Height))
+    const blockNumberProgressionValid = publicInput.blockNumber
+      .equals(proof1.publicInput.blockNumber)
+      .and(
+        proof1.publicOutput.blockNumber.equals(proof2.publicInput.blockNumber)
+      );
+
+    // For tx proofs, we check that the progression starts and end with MAX
+    // in addition to that both proofs are non-closed
+    const isValidTransactionMerge = publicInput.blockNumber
+      .equals(MAX_FIELD)
+      .and(blockNumberProgressionValid)
       .and(proof1Closed.or(proof2Closed).not());
 
     const isValidClosedMerge = proof1Closed
       .and(proof2Closed)
-      .and(proof1Height.add(1).equals(proof2Height));
+      .and(blockNumberProgressionValid);
 
     isValidTransactionMerge
       .or(isValidClosedMerge)
@@ -769,9 +779,8 @@ export class BlockProverProgrammable extends ZkProgrammable<
       blockHashRoot: proof2.publicOutput.blockHashRoot,
       eternalTransactionsHash: proof2.publicOutput.eternalTransactionsHash,
       incomingMessagesHash: proof2.publicOutput.incomingMessagesHash,
-      // Provable.if(isValidClosedMerge, Bool(true), Bool(false));
       closed: isValidClosedMerge,
-      blockNumber: proof2Height,
+      blockNumber: proof2.publicOutput.blockNumber,
     });
   }
 

packages/protocol/src/prover/statetransition/StateTransitionProvable.ts

@@ -3,8 +3,6 @@ import { WithZkProgrammable } from "@proto-kit/common";
 
 import { StateTransitionProvableBatch } from "../../model/StateTransitionProvableBatch";
 
-import { StateTransitionWitnessProviderReference } from "./StateTransitionWitnessProviderReference";
-
 export class StateTransitionProverPublicInput extends Struct({
   stateTransitionsHash: Field,
   protocolTransitionsHash: Field,
@@ -29,8 +27,6 @@ export interface StateTransitionProvable
     StateTransitionProverPublicInput,
     StateTransitionProverPublicOutput
   > {
-  witnessProviderReference: StateTransitionWitnessProviderReference;
-
   runBatch: (
     publicInput: StateTransitionProverPublicInput,
     batch: StateTransitionProvableBatch

packages/protocol/src/prover/statetransition/StateTransitionProver.ts

@@ -27,8 +27,6 @@ import {
   StateTransitionProverPublicInput,
   StateTransitionProverPublicOutput,
 } from "./StateTransitionProvable";
-import { StateTransitionWitnessProvider } from "./StateTransitionWitnessProvider";
-import { StateTransitionWitnessProviderReference } from "./StateTransitionWitnessProviderReference";
 
 const errors = {
   propertyNotMatching: (property: string, step: string) =>
@@ -75,8 +73,7 @@ export class StateTransitionProverProgrammable extends ZkProgrammable<
   StateTransitionProverPublicOutput
 > {
   public constructor(
-    private readonly stateTransitionProver: StateTransitionProver,
-    public readonly witnessProviderReference: StateTransitionWitnessProviderReference
+    private readonly stateTransitionProver: StateTransitionProver
   ) {
     super();
   }
@@ -143,14 +140,6 @@ export class StateTransitionProverProgrammable extends ZkProgrammable<
     ];
   }
 
-  private get witnessProvider(): StateTransitionWitnessProvider {
-    const provider = this.witnessProviderReference.getWitnessProvider();
-    if (provider === undefined) {
-      throw errors.noWitnessProviderSet();
-    }
-    return provider;
-  }
-
   /**
    * Applies the state transitions to the current stateRoot
    * and returns the new prover state
@@ -179,12 +168,19 @@ export class StateTransitionProverProgrammable extends ZkProgrammable<
 
     const transitions = transitionBatch.batch;
     const types = transitionBatch.transitionTypes;
+    const merkleWitness = transitionBatch.merkleWitnesses;
     for (
       let index = 0;
       index < constants.stateTransitionProverBatchSize;
       index++
     ) {
-      this.applyTransition(state, transitions[index], types[index], index);
+      this.applyTransition(
+        state,
+        transitions[index],
+        types[index],
+        merkleWitness[index],
+        index
+      );
     }
 
     return state;
@@ -198,13 +194,10 @@ export class StateTransitionProverProgrammable extends ZkProgrammable<
     state: StateTransitionProverExecutionState,
     transition: ProvableStateTransition,
     type: ProvableStateTransitionType,
+    merkleWitness: RollupMerkleTreeWitness,
     index = 0
   ) {
-    const witness = Provable.witness(RollupMerkleTreeWitness, () =>
-      this.witnessProvider.getWitness(Field(transition.path.toString()))
-    );
-
-    const membershipValid = witness.checkMembership(
+    const membershipValid = merkleWitness.checkMembership(
       state.stateRoot,
       transition.path,
       transition.from.value
@@ -214,7 +207,7 @@ export class StateTransitionProverProgrammable extends ZkProgrammable<
       .or(transition.from.isSome.not())
       .assertTrue(errors.merkleWitnessNotCorrect(index, type));
 
-    const newRoot = witness.calculateRoot(transition.to.value);
+    const newRoot = merkleWitness.calculateRoot(transition.to.value);
 
     state.stateRoot = Provable.if(
       transition.to.isSome,
@@ -349,15 +342,9 @@ export class StateTransitionProver
 {
   public zkProgrammable: StateTransitionProverProgrammable;
 
-  public constructor(
-    // Injected
-    public readonly witnessProviderReference: StateTransitionWitnessProviderReference
-  ) {
+  public constructor() {
     super();
-    this.zkProgrammable = new StateTransitionProverProgrammable(
-      this,
-      witnessProviderReference
-    );
+    this.zkProgrammable = new StateTransitionProverProgrammable(this);
   }
 
   public runBatch(