Merge pull request #260 from proto-kit/feature/st-prover-3

STProver v3

Author: maht0rz

.github/workflows/pull-request-develop.yml

@@ -80,7 +80,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 22.9.0
+          node-version: 18
           cache: npm
 
       - name: "Install dependencies"

package-lock.json

@@ -2,6 +2,7 @@ export * from "./config/ModuleContainer";
 export * from "./config/ConfigurableModule";
 export * from "./config/ChildContainerProvider";
 export * from "./config/ChildContainerCreatable";
+export * from "./config/injectAlias";
 export * from "./types";
 export * from "./zkProgrammable/ZkProgrammable";
 export * from "./zkProgrammable/ProvableMethodExecutionContext";
@@ -22,4 +23,3 @@ export * from "./compiling/AtomicCompileHelper";
 export * from "./compiling/CompileRegistry";
 export * from "./compiling/CompilableModule";
 export * from "./compiling/services/ChildVerificationKeyService";
-export * from "./config/injectAlias";

packages/common/src/index.ts

@@ -200,7 +200,7 @@ export function createMerkleTree(height: number): AbstractMerkleTreeClass {
 
     public static dummy() {
       return new RollupMerkleWitness({
-        isLeft: Array<Bool>(height - 1).fill(Bool(false)),
+        isLeft: Array<Bool>(height - 1).fill(Bool(true)),
         path: Array<Field>(height - 1).fill(Field(0)),
       });
     }
@@ -222,6 +222,7 @@ export function createMerkleTree(height: number): AbstractMerkleTreeClass {
     public static WITNESS = RollupMerkleWitness;
 
     // private in interface
+    // TODO Cache this in some static variable so that we don't recompute it every time
     readonly zeroes: bigint[];
 
     readonly store: MerkleTreeStore;

packages/common/src/trees/RollupMerkleTree.ts

@@ -5,6 +5,7 @@ import {
   DynamicProof,
   Proof,
 } from "o1js";
+import _ from "lodash";
 
 import { TypedClass } from "./types";
 
@@ -72,6 +73,26 @@ export function reduceSequential<T, U>(
   );
 }
 
+export function yieldSequential<Source, State, Target>(
+  array: Source[],
+  callbackfn: (
+    previousValue: State,
+    currentValue: Source,
+    currentIndex: number,
+    array: Source[]
+  ) => Promise<[State, Target]>,
+  initialValue: State
+): Promise<[State, Target[]]> {
+  return reduceSequential<Source, [State, Target[]]>(
+    array,
+    async ([state, collectedTargets], curr, index, arr) => {
+      const [newState, addition] = await callbackfn(state, curr, index, arr);
+      return [newState, collectedTargets.concat(addition)];
+    },
+    [initialValue, []]
+  );
+}
+
 export function mapSequential<T, R>(
   array: T[],
   f: (element: T, index: number, array: T[]) => Promise<R>
@@ -198,3 +219,43 @@ export function safeParseJson<T>(json: string) {
   // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
   return JSON.parse(json) as T;
 }
+
+export type Nullable<T> = {
+  [Key in keyof T]: T[Key] | undefined;
+};
+
+export function isFull<T>(t: Nullable<T>): t is T {
+  return Object.values(t).findIndex((v) => v === undefined) === -1;
+}
+
+// TODO Restructure utils into separate package and multiple files
+
+export function padArray<T>(
+  array: T[],
+  batchSize: number,
+  generator: (index: number) => T
+): T[] {
+  const slice = array.slice();
+  const dummies = range(0, batchSize - (array.length % batchSize)).map((i) =>
+    generator(i + array.length)
+  );
+  slice.push(...dummies);
+  return slice;
+}
+
+export function batch<T>(
+  arr: T[],
+  batchSize: number,
+  dummy: (index: number) => T
+): T[][] {
+  const padded = padArray(arr, batchSize, dummy);
+
+  const partitioned = _.groupBy(
+    padded.map((v, i) => [v, i] as const),
+    ([v, i]) => Math.floor(i / batchSize)
+  );
+
+  const numBatches = Math.ceil(arr.length / batchSize);
+
+  return range(0, numBatches).map((i) => partitioned[i].map((x) => x[0]));
+}

packages/common/src/utils.ts

@@ -31,7 +31,9 @@ export function toProver(
     );
 
     if (zkProgram === undefined) {
-      throw new Error("Correct ZkProgram not found");
+      throw new Error(
+        `Correct ZkProgram not found (searching for method ${methodName})`
+      );
     }
 
     if (areProofsEnabled) {

packages/common/src/zkProgrammable/provableMethod.ts

@@ -51,6 +51,7 @@ export class BullQueue
           await this.activePromise;
         }
         let resOutside: () => void = () => {};
+        // TODO Use Promise.withResolvers() for that
         const promise = new Promise<void>((res) => {
           resOutside = res;
         });

packages/deployment/src/queue/BullQueue.ts

@@ -6,11 +6,12 @@ import {
 } from "@proto-kit/module";
 import { inject, injectable } from "tsyringe";
 import {
+  BeforeTransactionHookArguments,
   ProvableTransactionHook,
-  BlockProverExecutionData,
   PublicKeyOption,
 } from "@proto-kit/protocol";
 import { Field, Provable, PublicKey } from "o1js";
+import { noop } from "@proto-kit/common";
 
 import { UInt64 } from "../math/UInt64";
 import { Balance, TokenId } from "../runtime/Balances";
@@ -122,8 +123,8 @@ export class TransactionFeeHook extends ProvableTransactionHook<TransactionFeeHo
    *
    * @param executionData
    */
-  public async onTransaction(
-    executionData: BlockProverExecutionData
+  public async beforeTransaction(
+    executionData: BeforeTransactionHookArguments
   ): Promise<void> {
     const feeConfig = Provable.witness(MethodFeeConfigData, () =>
       this.feeAnalyzer.getFeeConfig(
@@ -154,4 +155,8 @@ export class TransactionFeeHook extends ProvableTransactionHook<TransactionFeeHo
       UInt64.Unsafe.fromField(fee.value)
     );
   }
+
+  public async afterTransaction(): Promise<void> {
+    noop();
+  }
 }

packages/library/src/hooks/TransactionFeeHook.ts

@@ -1,7 +1,6 @@
 import { Bool, Field, FlexibleProvablePure, Poseidon } from "o1js";
 import { container } from "tsyringe";
 import {
-  StateTransition,
   ProvableStateTransition,
   MethodPublicOutput,
   RuntimeMethodExecutionContext,
@@ -40,7 +39,7 @@ const errors = {
 };
 
 export function toStateTransitionsHash(
-  stateTransitions: StateTransition<any>[]
+  stateTransitions: { toProvable: () => ProvableStateTransition }[]
 ) {
   const stateTransitionsHashList = new StateTransitionReductionList(
     ProvableStateTransition

packages/module/src/method/runtimeMethod.ts

@@ -0,0 +1,21 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `blockStateTransitions` on the `BlockResult` table. All the data in the column will be lost.
+  - You are about to drop the column `protocolTransitions` on the `TransactionExecutionResult` table. All the data in the column will be lost.
+  - Added the required column `beforeBlockStateTransitions` to the `Block` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `fromStateRoot` to the `Block` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `afterBlockStateTransitions` to the `BlockResult` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "Block" ADD COLUMN     "beforeBlockStateTransitions" JSON NOT NULL,
+ADD COLUMN     "fromStateRoot" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "BlockResult" DROP COLUMN "blockStateTransitions",
+ADD COLUMN     "afterBlockStateTransitions" JSON NOT NULL,
+ADD COLUMN     "witnessedRoots" TEXT[];
+
+-- AlterTable
+ALTER TABLE "TransactionExecutionResult" DROP COLUMN "protocolTransitions";