optimization pass. processing 1kb json now 65k constraints

Author: zac-williamson

out.txt

@@ -34,6 +34,70 @@ impl JSONEntry {
         // 4 gates. oof
     }
 
+    unconstrained fn __extract_parent_index_from_field(f: Field) -> (Field, Field, Field) {
+        let entry = JSONEntry::from_field(f);
+        let low = entry.id;
+        let mid = entry.parent_index;
+        let hi = (f - low - mid * 0x10000) / 0x100000000;
+        (low, mid, hi)
+    }
+    fn add_child_pointer_into_field(f: Field, child_pointer: Field) -> Field {
+        f + child_pointer * 0x1000000000000
+    }
+    unconstrained fn __extract_entry_type_and_id_from_field(f: Field) -> (Field, Field, Field) {
+        let entry = JSONEntry::from_field(f);
+        let id = entry.id;
+        let entry_type = entry.entry_type;
+        let mid = (f - id - entry_type * 0x100000000000000000000000000000000000000) / 0x10000;
+        (id, mid, entry_type)
+    }
+    unconstrained fn __extract_entry_type_id_and_parent_index_from_field(f: Field) -> (Field, Field, Field, Field) {
+        let entry = JSONEntry::from_field(f);
+        let id = entry.id;
+        let entry_type = entry.entry_type;
+        let parent_index = entry.parent_index;
+        let mid = (f - id - parent_index * 0x10000 - entry_type * 0x100000000000000000000000000000000000000)
+            / 0x100000000;
+        (id, parent_index, mid, entry_type)
+    }
+
+    // 11.75 gates
+    fn extract_entry_type_id_and_parent_index_from_field(f: Field) -> (Field, Field, Field) {
+        let (id, parent_index, mid, entry_type) = JSONEntry::__extract_entry_type_id_and_parent_index_from_field(f);
+        id.assert_max_bit_size(8); // 1.25
+        parent_index.assert_max_bit_size(16); // 1.5
+        entry_type.assert_max_bit_size(16); // 1.5
+        mid.assert_max_bit_size(136); // 5.5
+
+        assert(
+            id
+            + parent_index * 0x10000
+            + mid * 0x100000000
+            + entry_type * 0x100000000000000000000000000000000000000
+            == f
+        );
+
+        (id, parent_index, entry_type)
+    }
+    fn extract_entry_type_and_id_from_field(f: Field) -> (Field, Field) {
+        let (id, mid, entry_type) = JSONEntry::__extract_entry_type_and_id_from_field(f);
+        id.assert_max_bit_size(8); // 1.25
+        entry_type.assert_max_bit_size(16); // 1.5
+        mid.assert_max_bit_size(136); // 5.5
+
+        assert(id + mid * 0x10000 + entry_type * 0x100000000000000000000000000000000000000 == f);
+        (id, entry_type)
+    }
+    fn extract_parent_index_from_field(f: Field) -> Field {
+        let (low, parent_index, hi) = JSONEntry::__extract_parent_index_from_field(f);
+
+        low.assert_max_bit_size(16); // 1.75
+        hi.assert_max_bit_size(128); // 5.5
+        parent_index.assert_max_bit_size(16); // 1.75
+        assert(low + parent_index * 0x10000 + hi * 0x100000000 == f); // 1
+        // 10 gates?
+        parent_index
+    }
     fn from_field(f: Field) -> Self {
         let bytes: [u8; 20] = f.to_be_bytes(20).as_array(); // 10.5 gates
 

src/json_entry.nr

@@ -377,7 +377,7 @@ struct Hasher<let KeyFields: u16>
 
 impl<let KeyFields: u16> Hasher<KeyFields> {
 
-    fn get_keyhash<let NumPackedFields: u16>(_: Self, packed_fields: [Field; NumPackedFields], body_index: u16, key_length: u16) -> Field {
+    fn get_keyhash<let NumPackedFields: u16>(_: Self, packed_fields: [Field; NumPackedFields], body_index: Field, key_length: Field) -> Field {
         let key_fields: [Field; KeyFields] = slice_fields(packed_fields, body_index, key_length);
 
         let hashed_full = dep::std::hash::poseidon2::Poseidon2::hash(key_fields, KeyFields as u32);

src/keyhash.nr

@@ -3,8 +3,7 @@ use crate::json_entry::JSONEntry;
 use crate::lt::lt_field_16_bit;
 use crate::lt::lte_field_240_bit;
 use crate::lt::assert_lte_240_bit;
-use crate::redux::BEGIN_OBJECT_TOKEN;
-use crate::redux::BEGIN_ARRAY_TOKEN;
+use crate::redux_tables::{TOKEN_BEGINS_OBJECT_OR_ARRAY, BEGIN_OBJECT_TOKEN, BEGIN_ARRAY_TOKEN};
 use crate::keyhash::get_keyhash_chunky;
 use crate::keyhash::get_keyhash;
 use crate::keyhash::slice_200_bits_from_field;
@@ -15,8 +14,8 @@ use dep::noir_sort;
 
 use dep::std::hash::poseidon2;
 struct KeyIndexData {
-    json_index: u16,
-    json_length: u16,
+    json_index: Field,
+    json_length: Field,
     parent_id: Field,
     array_index: Field,
 }
@@ -32,155 +31,89 @@ impl KeyIndexData {
     fn from_field(packed: Field) -> Self {
         let unpacked = packed.to_be_bytes(8);
         let array_index: Field = unpacked[1] as Field + unpacked[0] as Field * 0x100;
-        let json_length: u16 = unpacked[3] as u16 + unpacked[2] as u16 * 0x100;
-        let json_index: u16 = unpacked[5] as u16 + unpacked[4] as u16 * 0x100;
+        let json_length: Field = unpacked[3] as Field + unpacked[2] as Field * 0x100;
+        let json_index: Field = unpacked[5] as Field + unpacked[4] as Field * 0x100;
         let parent_id: Field = unpacked[7] as Field + unpacked[6] as Field * 0x100;
         KeyIndexData { json_index, json_length, parent_id, array_index }
     }
 }
 
-fn fakehash<let N: u32>(input: [u8; N]) -> [u8; 32] {
-    let mut r = 0;
-    for i in 0..N {
-        r *= 0x100;
-        r += input[i] as Field;
-    }
-    r *= 0xffee134029374623874;
-    r.to_be_bytes(32).as_array()
-}
-
-global KeyLen = 32; // todo make param
-
-// global BYTE_MULTIPLIERS: [Field; 31] = [
-//     1,
-//     0x100,
-//     0x10000,
-//     0x1000000,
-//     0x100000000,
-//     0x10000000000,
-//     0x1000000000000,
-//     0x100000000000000,
-//     0x10000000000000
-// ]
-
 impl<let NumBytes: u32, let NumPackedFields: u16, let TranscriptEntries: u32> JSON<NumBytes, NumPackedFields, TranscriptEntries> {
-
-    // 101,105
-    // 56,876
-    // 44,229 cost
-    // 700 per iteration
-    // TODO: is poseidon2 cheap??? sounds like
     fn compute_keyhash_and_sort_json_entries(&mut self) {
         let hasher: Hasher<2> = Hasher {};
 
         let mut hashlist: [Field; TranscriptEntries] = [0; TranscriptEntries];
-        // 77321 - 73798 div 10 = 352?
-        // should be 287 ah bad hasher
+
         let two_pow_200 = 0x10000000000000000000000000000000000000000000000000000;
         let two_pow_216 = 0x100000000000000000000000000000000000000000000000000000000;
         for i in 0..TranscriptEntries {
             let KeyIndexData{ json_index, json_length, parent_id, array_index } = KeyIndexData::from_field(self.key_data[i]);
             let hash = hasher.get_keyhash(self.packed_json, json_index, json_length);
             hashlist[i] = hash + array_index * two_pow_200 + parent_id * two_pow_216;
-            std::as_witness(hashlist[i]);
         }
-        // ok the next pile of bullshit follows
-        // we need to sort the JSON entries accordingf to the keyhash sort pattern
-        // once we do that we can *finally* move on to extracting data from the json, maybe
-        //  = self.json_entries;
-        // TODO THIS SHOULD NOT BE HERE, MESSY
-        // let mut parent_indices: [Field; TranscriptEntries] = [0; TranscriptEntries];
-        // for i in 0..TranscriptEntries {
-        //     let E = self.json_entries[i];
-        //     if (E.child_pointer != 0) {
-        //         let child_idx = E.child_pointer;
-        //         let parent_identity = self.json_entries[child_idx].parent_index;
-        //         parent_indices[parent_identity] = child_idx;
-        //     }
-        // }
-        // we want a list that maps parent ID to json entry
-        // if I know that parent id 5 maps to idx 19 that is important
-        // ok so now we have, for ONE child, the location of the parent
-        //
 
         let sort_result = noir_sort::sort_advanced(hashlist, lte_field_240_bit, assert_lte_240_bit);
-        // about 2k to sort. no biggie
 
         let mut sorted_entries: [Field; TranscriptEntries] = [0; TranscriptEntries];
 
         for i in 0..TranscriptEntries {
             sorted_entries[sort_result.sort_indices[i]] = self.packed_json_entries[i];
         }
 
+        let mut ids: [Field; TranscriptEntries] = [0; TranscriptEntries];
+        let mut parent_indices: [Field; TranscriptEntries] = [0; TranscriptEntries];
+        let mut entry_types: [Field; TranscriptEntries] = [0; TranscriptEntries];
+
+        for i in 0..TranscriptEntries {
+            // 11.75 + 3.5 = 15.25 gates per iteration
+            let (id, parent_index, entry_type) = JSONEntry::extract_entry_type_id_and_parent_index_from_field(sorted_entries[i]);
+            ids[i] = id;
+            parent_indices[i] = parent_index;
+            entry_types[i] = entry_type;
+        }
+
         let mut identity_to_json_map: [Field; TranscriptEntries] = [0; TranscriptEntries];
+        // 6.5 gates per iteration
         for i in 0..TranscriptEntries {
-            let E = JSONEntry::from_field(sorted_entries[i]);
-            let update = ((E.entry_type == BEGIN_OBJECT_TOKEN) | (E.entry_type == BEGIN_ARRAY_TOKEN));
+            let id = ids[i];
+            let entry_type = entry_types[i];
+            // 2 gates
+            let update = TOKEN_BEGINS_OBJECT_OR_ARRAY[entry_type];
             // NOTE THIS RELIES ON TRANSCRIPTENTRIES ACTUALLY DESCRIBING NUMTRANSCRIPTENTRIES + 1
-            let index = (E.id - (TranscriptEntries as Field - 1)) * update as Field
-                + (TranscriptEntries as Field - 1);
+            // 1 gate
+            let index = (id - (TranscriptEntries as Field - 1)) * update + (TranscriptEntries as Field - 1);
+            // 3.5 gates
             identity_to_json_map[index] = i as Field;
-            // if ((E.entry_type == BEGIN_OBJECT_TOKEN) | (E.entry_type == BEGIN_ARRAY_TOKEN)) {
-            //     identity_to_json_map[E.id] = i as Field;
-            // }
         }
-        // 67,802
-        // 105,261
-        // almost 40k?
-
-        // this one is expensive... if statement!
-        /*
-            ok wtf is going on here
-            a json entry has a "parent_index"
-            if the parent index changes, we are changing context
-            if this happens then we need to find the owner of this new entry
-            and update its "child pointer" location
-        */
-        for i in 0..TranscriptEntries - 1 {
-            // 35 gates per unpack = 105 gates per iteration
-            // 110 * 64 = 6500 ish
-            let parent_identity_pre = JSONEntry::from_field(sorted_entries[i]).parent_index;
-            let parent_identity_post = JSONEntry::from_field(sorted_entries[i + 1]).parent_index;
+
+        // 13.5 gates per iteration
+        let mut parent_identity_pre = parent_indices[0];
+        for i in 1..TranscriptEntries {
+            let parent_identity_post = parent_indices[i];
             // if the parent identity changes,
-            let new_parent = parent_identity_post != parent_identity_pre;
 
+            // the list is sorted according to parent_ideneity,
+            // so parent_identity increments in steps of 0 or 1
+            // 1 gate
+            let new_parent = parent_identity_post - parent_identity_pre;
+
+            // 3.5 gates
             let index_of_parent = identity_to_json_map[parent_identity_post];
-            let mut updated = JSONEntry::from_field(sorted_entries[index_of_parent]);
-            updated.child_pointer = i as Field + 1;
-
-            // // RELIES ON THE SMALLEST ENTRY IN THE SORTED LIST BEING EMPTY
-            let index = ((index_of_parent - 0) * new_parent as Field) + 0;
-            sorted_entries[index] = updated.to_field();
-            // if (new_parent) {
-            //     let index_of_parent = identity_to_json_map[parent_identity_post];
-            //     let mut updated = JSONEntry::from_field(sorted_entries[index_of_parent]);
-            //     updated.child_pointer = i as Field + 1;
-            //     sorted_entries[index_of_parent] = updated.to_field();
-            // }
-            // i + 1 is the starting index of a new set of children
+            // 1 gate + 3.5 gates
+            let updated = JSONEntry::add_child_pointer_into_field(sorted_entries[index_of_parent], i as Field);
+
+            // RELIES ON THE SMALLEST ENTRY IN THE SORTED LIST BEING EMPTY
+            // 1 gate
+            let index = (index_of_parent * new_parent);
+            // 3.5 gates
+            sorted_entries[index] = updated;
+
+            parent_identity_pre = parent_identity_post;
         }
 
-        // phew need to consolidate.
-        // 1: throw unused code in a scrapbook
-        // 2: tidy data structures and remove unused fluff
-        // 3: add metadata to JSON objeect so that we can query internal parts
-
-        // 4: do we need a different interface for a JSON Object vs JSON Array?
-        // for i in 0..TranscriptEntries {
-        //     let old_child_pointer = sorted_entries[i].child_pointer;
-
-        //     let new_child_pointer = sort_result.sort_indices[old_child_pointer];
-        //     // TODO: hacky workaround, fix
-        //     if (old_child_pointer != 0) {
-        //         sorted_entries[i].child_pointer = new_child_pointer;
-        //     }
-        // }
         self.packed_json_entries = sorted_entries;
-
         self.key_hashes = sort_result.sorted;
-        // 38050
-        // 60707 <-- cost after hashing keuys
-        // 77244 <-- cost after sorting and updating children
     }
 }
 
+// 68002

src/keymap.nr

@@ -304,7 +304,7 @@ fn mainslice(foo: [Field; 100], bar: [u8; 1000], num_bytes: [u16; 100], start_by
 
     // CURRENT VALUE OF SLICE_FIELDS IS 182 GATES 26 AUGUST
     for i in 0..89 {
-        let x: [Field; 2] = slice_fields(foo, start_bytes[i], num_bytes[i]);
+        let x: [Field; 2] = slice_fields(foo, start_bytes[i] as Field, num_bytes[i] as Field);
         // let x = slice_field(foo[i], start_bytes[i] as Field);
         println(f"{x}");
         //  assert(x[0] == x[1]);

src/main.nr

@@ -21,6 +21,9 @@ global OBJECT_LAYER = 0;
 global ARRAY_LAYER = 1;
 global SINGLE_VALUE_LAYER = 2;
 
+global TOKEN_BEGINS_OBJECT_OR_ARRAY : [Field; 11] = [
+0,1,0,1,0,0,0,0,0,0,0
+];
 global TOKEN_IS_NUMERIC_OR_LITERAL: [Field; 11] = [
 0,0,0,0,0,0,0,0,1,1,0
 ];