Merge pull request #2632 from o1-labs/querolita/napi-proof

[WIP] (`o1js`) Native Prover with Napi - Proof

Author: querolita

src/bindings/crypto/bindings.ts

@@ -3,6 +3,7 @@
  * It is exposed to JSOO by populating a global variable with an object.
  * It gets imported as the first thing in ../../bindings.js so that the global variable is ready by the time JSOO code gets executed.
  */
+import type * as napiNamespace from '../compiled/node_bindings/plonk_wasm.cjs';
 import type * as wasmNamespace from '../compiled/node_bindings/plonk_wasm.cjs';
 import { prefixHashes, prefixHashesLegacy } from '../crypto/constants.js';
 import { Bigint256Bindings } from './bindings/bigint256.js';
@@ -14,12 +15,15 @@ import { verifierIndexConversion } from './bindings/conversion-verifier-index.js
 import { PallasBindings, VestaBindings } from './bindings/curve.js';
 import { jsEnvironment } from './bindings/env.js';
 import { FpBindings, FqBindings } from './bindings/field.js';
-import { srs } from './bindings/srs.js';
 import { FpVectorBindings, FqVectorBindings } from './bindings/vector.js';
+import { srs } from './bindings/srs.js';
+import { srs as napiSrs } from './napi-srs.js';
 import { napiConversionCore } from './napi-conversion-core.js';
 import { napiProofConversion } from './napi-conversion-proof.js';
+import { napiVerifierIndexConversion } from './napi-conversion-verifier-index.js';
+import { napiOraclesConversion } from './bindings/napi-conversion-oracles.js';
 
-export { RustConversion, Wasm, createNativeRustConversion, getRustConversion };
+export { Napi, RustConversion, Wasm, createNativeRustConversion, getRustConversion };
 
 /* TODO: Uncomment in phase 2 of conversion layer 
 import { conversionCore as conversionCoreNative } from './native/conversion-core.js';
@@ -49,13 +53,14 @@ const tsBindings = {
     return bundle.srsFactory(wasm, bundle.conversion);
   },*/
   srs: (wasm: Wasm) => srs(wasm, getRustConversion(wasm)),
-  srsNative: (napi: Wasm) => srs(napi, createNativeRustConversion(napi) as any),
+  srsNative: (napi: Napi) => napiSrs(napi, createNativeRustConversion(napi) as any),
 };
 
 // this is put in a global variable so that mina/src/lib/crypto/kimchi_bindings/js/bindings.js finds it
 (globalThis as any).__snarkyTsBindings = tsBindings;
 
 type Wasm = typeof wasmNamespace;
+type Napi = typeof napiNamespace;
 
 type RustConversion = ReturnType<typeof buildWasmConversion>;
 
@@ -86,9 +91,11 @@ function buildWasmConversion(wasm: Wasm) {
 function createNativeRustConversion(napi: any) {
   let core = napiConversionCore(napi);
   let proof = napiProofConversion(napi, core);
+  let verif = napiVerifierIndexConversion(napi, core);
+  let oracles = napiOraclesConversion(napi);
   return {
-    fp: { ...core.fp, ...proof.fp },
-    fq: { ...core.fq, ...proof.fq },
+    fp: { ...core.fp, ...proof.fp, ...verif.fp, ...oracles.fp },
+    fq: { ...core.fq, ...proof.fq, ...verif.fq, ...oracles.fq },
   };
 }
 

src/bindings/crypto/bindings/conversion-base.ts

@@ -1,24 +1,24 @@
-import { Field } from './field.js';
-import { bigintToBytes32, bytesToBigint32 } from '../bigint-helpers.js';
+import type { MlArray } from '../../../lib/ml/base.js';
 import type {
   WasmGPallas,
   WasmGVesta,
   WasmPallasGProjective,
   WasmVestaGProjective,
 } from '../../compiled/node_bindings/plonk_wasm.cjs';
-import type { MlArray } from '../../../lib/ml/base.js';
-import { OrInfinity, Infinity } from './curve.js';
+import { bigintToBytes32, bytesToBigint32 } from '../bigint-helpers.js';
+import { Infinity, OrInfinity } from './curve.js';
+import { Field } from './field.js';
 
 export {
-  fieldToRust,
+  WasmAffine,
+  WasmProjective,
+  affineFromRust,
+  affineToRust,
   fieldFromRust,
-  fieldsToRustFlat,
+  fieldToRust,
   fieldsFromRustFlat,
+  fieldsToRustFlat,
   maybeFieldToRust,
-  affineToRust,
-  affineFromRust,
-  WasmAffine,
-  WasmProjective,
 };
 
 // TODO: Hardcoding this is a little brittle

src/bindings/crypto/bindings/conversion-core.ts

@@ -116,9 +116,9 @@ function conversionCorePerField(
       return new PolyComm(rustUnshifted, rustShifted);
     },
     polyCommFromRust(polyComm: WasmPolyComm): PolyComm {
-      console.log('polyComm', polyComm);
+      console.log('polyComm old', polyComm);
       let rustUnshifted = polyComm.unshifted;
-      console.log('polyCommFromRust', rustUnshifted);
+      console.log('rustUnshifted', rustUnshifted);
       let mlUnshifted = mapFromUintArray(rustUnshifted, (ptr) => {
         return affineFromRust(wrap(ptr, CommitmentCurve));
       });

src/bindings/crypto/bindings/conversion-proof.ts

@@ -1,46 +1,46 @@
+import { MlArray, MlOption, MlTuple } from '../../../lib/ml/base.js';
+import type * as wasmNamespace from '../../compiled/node_bindings/plonk_wasm.cjs';
 import type {
   WasmFpLookupCommitments,
-  WasmPastaFpLookupTable,
   WasmFpOpeningProof,
   WasmFpProverCommitments,
   WasmFpProverProof,
   WasmFpRuntimeTable,
-  WasmPastaFpRuntimeTableCfg,
   WasmFqLookupCommitments,
   WasmFqOpeningProof,
   WasmFqProverCommitments,
-  WasmPastaFqLookupTable,
   WasmFqProverProof,
   WasmFqRuntimeTable,
+  WasmPastaFpLookupTable,
+  WasmPastaFpRuntimeTableCfg,
+  WasmPastaFqLookupTable,
   WasmPastaFqRuntimeTableCfg,
   WasmVecVecFp,
   WasmVecVecFq,
 } from '../../compiled/node_bindings/plonk_wasm.cjs';
-import type * as wasmNamespace from '../../compiled/node_bindings/plonk_wasm.cjs';
+import {
+  fieldFromRust,
+  fieldToRust,
+  fieldsFromRustFlat,
+  fieldsToRustFlat,
+} from './conversion-base.js';
+import { ConversionCore, ConversionCores, mapToUint32Array, unwrap } from './conversion-core.js';
 import type {
+  Field,
+  LookupCommitments,
+  LookupTable,
+  OpeningProof,
   OrInfinity,
   PointEvaluations,
   PolyComm,
-  ProverProof,
-  ProofWithPublic,
   ProofEvaluations,
+  ProofWithPublic,
   ProverCommitments,
-  OpeningProof,
+  ProverProof,
   RecursionChallenge,
-  LookupCommitments,
   RuntimeTable,
   RuntimeTableCfg,
-  LookupTable,
-  Field,
 } from './kimchi-types.js';
-import { MlArray, MlOption, MlTuple } from '../../../lib/ml/base.js';
-import {
-  fieldToRust,
-  fieldFromRust,
-  fieldsToRustFlat,
-  fieldsFromRustFlat,
-} from './conversion-base.js';
-import { ConversionCore, ConversionCores, mapToUint32Array, unwrap } from './conversion-core.js';
 
 export { proofConversion };
 
@@ -178,6 +178,7 @@ function proofConversionPerField(
   }
 
   function runtimeTableToRust([, id, data]: RuntimeTable): WasmRuntimeTable {
+    console.log('old runtime table to rust!');
     return new RuntimeTable(id, core.vectorToRust(data));
   }
 

src/bindings/crypto/bindings/conversion-verifier-index.ts

@@ -117,8 +117,6 @@ function verifierIndexConversionPerField(
     );
   }
   function verificationEvalsFromRust(evals: WasmVerificationEvals): VerificationEvals {
-    console.log('evals', evals.coefficients_comm);
-
     let mlEvals: VerificationEvals = [
       0,
       core.polyCommsFromRust(evals.sigma_comm),

src/bindings/crypto/bindings/gate-vector-napi.unit-test.ts

@@ -6,9 +6,11 @@ import type { Field, Gate, Wire } from './kimchi-types.js';
 const require = createRequire(import.meta.url);
 
 function loadNative() {
+  const slug = `${process.platform}-${process.arch}`;
   const candidates = [
-    '../../compiled/_node_bindings/plonk_napi.node',
+    `../../../../../native/${slug}/plonk_napi.node`,
     '../../compiled/node_bindings/plonk_napi.node',
+    '../../compiled/_node_bindings/plonk_napi.node',
   ];
   for (const path of candidates) {
     try {
@@ -23,6 +25,21 @@ function loadNative() {
 
 const native: any = loadNative();
 
+const gateVectorCreate =
+  native.caml_pasta_fp_plonk_gate_vector_create ?? native.camlPastaFpPlonkGateVectorCreate;
+const gateVectorLen =
+  native.caml_pasta_fp_plonk_gate_vector_len ?? native.camlPastaFpPlonkGateVectorLen;
+const gateVectorAdd =
+  native.caml_pasta_fp_plonk_gate_vector_add ?? native.camlPastaFpPlonkGateVectorAdd;
+const gateVectorGet =
+  native.caml_pasta_fp_plonk_gate_vector_get ?? native.camlPastaFpPlonkGateVectorGet;
+const gateVectorWrap =
+  native.caml_pasta_fp_plonk_gate_vector_wrap ?? native.camlPastaFpPlonkGateVectorWrap;
+const gateVectorDigest =
+  native.caml_pasta_fp_plonk_gate_vector_digest ?? native.camlPastaFpPlonkGateVectorDigest;
+const circuitSerialize =
+  native.caml_pasta_fp_plonk_circuit_serialize ?? native.camlPastaFpPlonkCircuitSerialize;
+
 const { fp } = napiConversionCore(native);
 
 const zeroField: Field = [0, 0n];
@@ -44,24 +61,24 @@ const sampleGate: Gate = [
   [0, zeroField, zeroField, zeroField, zeroField, zeroField, zeroField, zeroField],
 ];
 
-const vector = native.camlPastaFpPlonkGateVectorCreate();
-expect(native.camlPastaFpPlonkGateVectorLen(vector)).toBe(0);
+const vector = gateVectorCreate();
+expect(gateVectorLen(vector)).toBe(0);
 
-native.camlPastaFpPlonkGateVectorAdd(vector, fp.gateToRust(sampleGate));
-expect(native.camlPastaFpPlonkGateVectorLen(vector)).toBe(1);
+gateVectorAdd(vector, fp.gateToRust(sampleGate));
+expect(gateVectorLen(vector)).toBe(1);
 
-const gate0 = native.camlPastaFpPlonkGateVectorGet(vector, 0);
+const gate0 = gateVectorGet(vector, 0);
 expect(gate0.typ).toBe(sampleGate[1]);
 
 const rustTarget = fp.wireToRust(mlWire(0, 0));
 const rustHead = fp.wireToRust(mlWire(1, 2));
-native.camlPastaFpPlonkGateVectorWrap(vector, rustTarget, rustHead);
-const wrapped = native.camlPastaFpPlonkGateVectorGet(vector, 0);
+gateVectorWrap(vector, rustTarget, rustHead);
+const wrapped = gateVectorGet(vector, 0);
 expect(wrapped.wires.w0).toEqual({ row: 1, col: 2 });
 
-native.camlPastaFpPlonkGateVectorDigest(0, vector);
-native.camlPastaFpPlonkCircuitSerialize(0, vector);
+gateVectorDigest(0, vector);
+circuitSerialize(0, vector);
 
-console.log('{}', native.camlPastaFpPlonkGateVectorDigest(0, vector));
+console.log('{}', gateVectorDigest(0, vector));
 
 console.log('gate vector napi bindings (fp) are working ✔️');

src/bindings/crypto/bindings/napi-conversion-oracles.ts

@@ -0,0 +1,123 @@
+import { MlOption } from '../../../lib/ml/base.js';
+import type * as napiNamespace from '../../compiled/node_bindings/plonk_wasm.cjs';
+import type {
+  WasmFpOracles,
+  WasmFpRandomOracles,
+  WasmFqOracles,
+  WasmFqRandomOracles,
+} from '../../compiled/node_bindings/plonk_wasm.cjs';
+import {
+  fieldFromRust,
+  fieldToRust,
+  fieldsFromRustFlat,
+  fieldsToRustFlat,
+  maybeFieldToRust,
+} from './conversion-base.js';
+import { Field, Oracles, RandomOracles, ScalarChallenge } from './kimchi-types.js';
+
+export { napiOraclesConversion };
+
+type napi = typeof napiNamespace;
+
+type NapiRandomOracles = WasmFpRandomOracles | WasmFqRandomOracles;
+type NapiOracles = WasmFpOracles | WasmFqOracles;
+
+type NapiClasses = {
+  RandomOracles: typeof WasmFpRandomOracles | typeof WasmFqRandomOracles;
+  Oracles: typeof WasmFpOracles | typeof WasmFqOracles;
+};
+
+function napiOraclesConversion(napi: napi) {
+  return {
+    fp: oraclesConversionPerField({
+      RandomOracles: napi.WasmFpRandomOracles,
+      Oracles: napi.WasmFpOracles,
+    }),
+    fq: oraclesConversionPerField({
+      RandomOracles: napi.WasmFqRandomOracles,
+      Oracles: napi.WasmFqOracles,
+    }),
+  };
+}
+
+function oraclesConversionPerField({ RandomOracles, Oracles }: NapiClasses) {
+  function randomOraclesToRust(ro: RandomOracles): NapiRandomOracles {
+    let jointCombinerMl = MlOption.from(ro[1]);
+    let jointCombinerChal = maybeFieldToRust(jointCombinerMl?.[1][1]);
+    let jointCombiner = maybeFieldToRust(jointCombinerMl?.[2]);
+    let beta = fieldToRust(ro[2]);
+    let gamma = fieldToRust(ro[3]);
+    let alphaChal = fieldToRust(ro[4][1]);
+    let alpha = fieldToRust(ro[5]);
+    let zeta = fieldToRust(ro[6]);
+    let v = fieldToRust(ro[7]);
+    let u = fieldToRust(ro[8]);
+    let zetaChal = fieldToRust(ro[9][1]);
+    let vChal = fieldToRust(ro[10][1]);
+    let uChal = fieldToRust(ro[11][1]);
+    return new RandomOracles(
+      jointCombinerChal,
+      jointCombiner,
+      beta,
+      gamma,
+      alphaChal,
+      alpha,
+      zeta,
+      v,
+      u,
+      zetaChal,
+      vChal,
+      uChal
+    );
+  }
+  function randomOraclesFromRust(ro: NapiRandomOracles): RandomOracles {
+    let jointCombinerChal = ro.joint_combiner_chal;
+    let jointCombiner = ro.joint_combiner;
+    let jointCombinerOption = MlOption<[0, ScalarChallenge, Field]>(
+      jointCombinerChal &&
+        jointCombiner && [0, [0, fieldFromRust(jointCombinerChal)], fieldFromRust(jointCombiner)]
+    );
+    let mlRo: RandomOracles = [
+      0,
+      jointCombinerOption,
+      fieldFromRust(ro.beta),
+      fieldFromRust(ro.gamma),
+      [0, fieldFromRust(ro.alpha_chal)],
+      fieldFromRust(ro.alpha),
+      fieldFromRust(ro.zeta),
+      fieldFromRust(ro.v),
+      fieldFromRust(ro.u),
+      [0, fieldFromRust(ro.zeta_chal)],
+      [0, fieldFromRust(ro.v_chal)],
+      [0, fieldFromRust(ro.u_chal)],
+    ];
+    // TODO: do we not want to free?
+    // ro.free();
+    return mlRo;
+  }
+
+  return {
+    oraclesToRust(oracles: Oracles): NapiOracles {
+      let [, o, pEval, openingPrechallenges, digestBeforeEvaluations] = oracles;
+      return new Oracles(
+        randomOraclesToRust(o),
+        fieldToRust(pEval[1]),
+        fieldToRust(pEval[2]),
+        fieldsToRustFlat(openingPrechallenges),
+        fieldToRust(digestBeforeEvaluations)
+      );
+    },
+    oraclesFromRust(oracles: NapiOracles): Oracles {
+      let mlOracles: Oracles = [
+        0,
+        randomOraclesFromRust(oracles.o),
+        [0, fieldFromRust(oracles.p_eval0), fieldFromRust(oracles.p_eval1)],
+        fieldsFromRustFlat(oracles.opening_prechallenges),
+        fieldFromRust(oracles.digest_before_evaluations),
+      ];
+      // TODO: do we not want to free?
+      // oracles.free();
+      return mlOracles;
+    },
+  };
+}

src/bindings/crypto/bindings/srs.ts

@@ -104,7 +104,7 @@ function srsPerField(f: 'fp' | 'fq', wasm: Wasm, conversion: RustConversion) {
   let maybeLagrangeCommitment = (srs: WasmSrs, domain_size: number, i: number) => {
     try {
       console.log(3);
-      console.log('srs', srs);
+      console.log('srs wasm', srs);
       let bytes = (wasm as any)[`caml_${f}_srs_to_bytes_external`](srs);
       console.log('bytes', bytes);
       let wasmSrs = undefined;