Merge branch 'develop' into fix/atomic-block-production

# Conflicts:
#	packages/sequencer/test/integration/BlockProduction.test.ts
#	packages/sequencer/test/settlement/Settlement.ts

Author: rpanic

.github/workflows/pull-request-develop.yml

@@ -91,6 +91,8 @@ jobs:
 
       - name: "Test"
         run: npm run test:ci
+        env:
+          IN_CI: true
 
   integration:
     runs-on: ubuntu-latest

.nvmrc

@@ -1 +1 @@
-v18.14.2
+v22.9.0

package-lock.json

@@ -34,7 +34,7 @@
     "@proto-kit/module": "*",
     "@proto-kit/protocol": "*",
     "@proto-kit/sequencer": "*",
-    "o1js": "^1.1.0",
+    "o1js": "^1.6.0",
     "tsyringe": "^4.7.0"
   },
   "devDependencies": {

packages/api/package.json

@@ -21,7 +21,7 @@ export class ComputedBlockModel {
       blockHashes.map(
         (blockHash) => blocks.find((block) => block?.hash === blockHash)!
       ),
-      proof.proof === MOCK_PROOF ? "mock-proof" : JSON.stringify(proof)
+      proof.proof === MOCK_PROOF ? MOCK_PROOF : JSON.stringify(proof)
     );
   }
 

packages/api/src/graphql/modules/BatchStorageResolver.ts

@@ -23,7 +23,7 @@
     "typescript-memoize": "^1.1.1"
   },
   "peerDependencies": {
-    "o1js": "^1.1.0",
+    "o1js": "^1.6.0",
     "tsyringe": "^4.7.0"
   },
   "devDependencies": {

packages/common/package.json

@@ -0,0 +1,62 @@
+import {
+  AreProofsEnabled,
+  CompileArtifact,
+  MOCK_VERIFICATION_KEY,
+} from "../zkProgrammable/ZkProgrammable";
+import { isSubtypeOfName } from "../utils";
+import { TypedClass } from "../types";
+import { log } from "../log";
+
+export type ArtifactRecord = Record<string, CompileArtifact>;
+
+export type CompileTarget = {
+  name: string;
+  compile: () => Promise<CompileArtifact>;
+};
+
+export class AtomicCompileHelper {
+  public constructor(private readonly areProofsEnabled: AreProofsEnabled) {}
+
+  private compilationPromises: {
+    [key: string]: Promise<CompileArtifact>;
+  } = {};
+
+  public async compileContract(
+    contract: CompileTarget,
+    overrideProofsEnabled?: boolean
+  ): Promise<CompileArtifact> {
+    let newPromise = false;
+    const { name } = contract;
+    if (this.compilationPromises[name] === undefined) {
+      const proofsEnabled =
+        overrideProofsEnabled ?? this.areProofsEnabled.areProofsEnabled;
+
+      // We only care about proofs enabled here if it's a contract, because
+      // in all other cases, ZkProgrammable already handles this switch, and we
+      // want to preserve the artifact layout (which might be more than one
+      // entry for ZkProgrammables)
+      if (
+        proofsEnabled ||
+        !isSubtypeOfName(
+          // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+          contract as unknown as TypedClass<any>,
+          "SmartContract"
+        )
+      ) {
+        log.time(`Compiling ${name}`);
+        this.compilationPromises[name] = contract.compile();
+        newPromise = true;
+      } else {
+        log.trace(`Compiling ${name} - mock`);
+        this.compilationPromises[name] = Promise.resolve({
+          verificationKey: MOCK_VERIFICATION_KEY,
+        });
+      }
+    }
+    const result = await this.compilationPromises[name];
+    if (newPromise) {
+      log.timeEnd.info(`Compiling ${name}`);
+    }
+    return result;
+  }
+}

packages/common/src/compiling/AtomicCompileHelper.ts

@@ -0,0 +1,6 @@
+import type { CompileRegistry } from "./CompileRegistry";
+import type { ArtifactRecord } from "./AtomicCompileHelper";
+
+export interface CompilableModule {
+  compile(registry: CompileRegistry): Promise<ArtifactRecord | void>;
+}

packages/common/src/compiling/CompilableModule.ts

@@ -0,0 +1,78 @@
+import { inject, injectable, singleton } from "tsyringe";
+
+import {
+  AreProofsEnabled,
+  CompileArtifact,
+} from "../zkProgrammable/ZkProgrammable";
+
+import {
+  ArtifactRecord,
+  AtomicCompileHelper,
+  CompileTarget,
+} from "./AtomicCompileHelper";
+
+/**
+ * The CompileRegistry compiles "compilable modules"
+ * (i.e. zkprograms, contracts or contractmodules)
+ * while making sure they don't get compiled twice in the same process in parallel.
+ */
+@injectable()
+@singleton()
+export class CompileRegistry {
+  public constructor(
+    @inject("AreProofsEnabled")
+    private readonly areProofsEnabled: AreProofsEnabled
+  ) {
+    this.compiler = new AtomicCompileHelper(this.areProofsEnabled);
+  }
+
+  private compiler: AtomicCompileHelper;
+
+  private artifacts: ArtifactRecord = {};
+
+  private inForceProverBlock = false;
+
+  /**
+   * This function forces compilation even if the artifact itself is in the registry.
+   * Basically the statement is: The artifact along is not enough, we need to
+   * actually have the prover compiled.
+   * This is true for non-sideloaded circuit dependencies.
+   */
+  public async forceProverExists(
+    f: (registry: CompileRegistry) => Promise<void>
+  ) {
+    this.inForceProverBlock = true;
+    await f(this);
+    this.inForceProverBlock = false;
+  }
+
+  public async compile(target: CompileTarget) {
+    if (this.artifacts[target.name] === undefined || this.inForceProverBlock) {
+      const artifact = await this.compiler.compileContract(target);
+      this.artifacts[target.name] = artifact;
+      return artifact;
+    }
+    return this.artifacts[target.name];
+  }
+
+  public getArtifact(name: string): CompileArtifact | undefined {
+    if (this.artifacts[name] === undefined) {
+      throw new Error(
+        `Artifact for ${name} not available, did you compile it via the CompileRegistry?`
+      );
+    }
+
+    return this.artifacts[name];
+  }
+
+  public addArtifactsRaw(artifacts: ArtifactRecord) {
+    this.artifacts = {
+      ...this.artifacts,
+      ...artifacts,
+    };
+  }
+
+  public getAllArtifacts() {
+    return this.artifacts;
+  }
+}

packages/common/src/compiling/CompileRegistry.ts

@@ -0,0 +1,26 @@
+import { injectable, Lifecycle, scoped } from "tsyringe";
+
+import { CompileRegistry } from "../CompileRegistry";
+
+@injectable()
+@scoped(Lifecycle.ContainerScoped)
+export class ChildVerificationKeyService {
+  private compileRegistry?: CompileRegistry;
+
+  public setCompileRegistry(registry: CompileRegistry) {
+    this.compileRegistry = registry;
+  }
+
+  public getVerificationKey(name: string) {
+    if (this.compileRegistry === undefined) {
+      throw new Error("CompileRegistry hasn't been set yet");
+    }
+    const artifact = this.compileRegistry.getArtifact(name);
+    if (artifact === undefined) {
+      throw new Error(
+        `Verification Key for child program ${name} not found in registry`
+      );
+    }
+    return artifact.verificationKey;
+  }
+}