changing main loop to not update "child pointer" - performing this when sorting in map

Author: zac-williamson

out.txt

@@ -47,6 +47,13 @@ impl JSONEntry {
         let parent_index = bytes[16] as Field * 0x100 + bytes[17] as Field; // 6 gates
         let id = bytes[18] as Field * 0x100 + bytes[19] as Field; // 6 gates
 
+        std::as_witness(json_length);
+        std::as_witness(json_pointer);
+        std::as_witness(num_children);
+        std::as_witness(child_pointer);
+        std::as_witness(array_pointer);
+        std::as_witness(parent_index);
+        std::as_witness(id);
         // this might cost 17 gates? oof
         JSONEntry { array_pointer, child_pointer, num_children, json_pointer, json_length, entry_type, parent_index, id }
     }

src/json_entry.nr

@@ -104,6 +104,8 @@ impl<let NumBytes: u32, let NumPackedFields: u16, let TranscriptEntries: u32> JS
         //
 
         let sort_result = noir_sort::sort_advanced(hashlist, lte_field_240_bit, assert_lte_240_bit);
+        // about 2k to sort. no biggie
+
         let mut sorted_entries: [Field; TranscriptEntries] = [0; TranscriptEntries];
 
         for i in 0..TranscriptEntries {
@@ -113,36 +115,48 @@ impl<let NumBytes: u32, let NumPackedFields: u16, let TranscriptEntries: u32> JS
         let mut identity_to_json_map: [Field; TranscriptEntries] = [0; TranscriptEntries];
         for i in 0..TranscriptEntries {
             let E = JSONEntry::from_field(sorted_entries[i]);
-            //let update = ((E.entry_type == BEGIN_OBJECT_TOKEN) | (E.entry_type == BEGIN_ARRAY_TOKEN));
+            let update = ((E.entry_type == BEGIN_OBJECT_TOKEN) | (E.entry_type == BEGIN_ARRAY_TOKEN));
             // NOTE THIS RELIES ON TRANSCRIPTENTRIES ACTUALLY DESCRIBING NUMTRANSCRIPTENTRIES + 1
-            // let index = (E.id - (TranscriptEntries as Field - 1)) * update as Field
-            //     + (TranscriptEntries as Field - 1);
-            // identity_to_json_map[index] = i as Field;
-            if ((E.entry_type == BEGIN_OBJECT_TOKEN) | (E.entry_type == BEGIN_ARRAY_TOKEN)) {
-                identity_to_json_map[E.id] = i as Field;
-            }
+            let index = (E.id - (TranscriptEntries as Field - 1)) * update as Field
+                + (TranscriptEntries as Field - 1);
+            identity_to_json_map[index] = i as Field;
+            // if ((E.entry_type == BEGIN_OBJECT_TOKEN) | (E.entry_type == BEGIN_ARRAY_TOKEN)) {
+            //     identity_to_json_map[E.id] = i as Field;
+            // }
         }
-
+        // 67,802
+        // 105,261
+        // almost 40k?
+
+        // this one is expensive... if statement!
+        /*
+            ok wtf is going on here
+            a json entry has a "parent_index"
+            if the parent index changes, we are changing context
+            if this happens then we need to find the owner of this new entry
+            and update its "child pointer" location
+        */
         for i in 0..TranscriptEntries - 1 {
+            // 35 gates per unpack = 105 gates per iteration
+            // 110 * 64 = 6500 ish
             let parent_identity_pre = JSONEntry::from_field(sorted_entries[i]).parent_index;
             let parent_identity_post = JSONEntry::from_field(sorted_entries[i + 1]).parent_index;
             // if the parent identity changes,
             let new_parent = parent_identity_post != parent_identity_pre;
 
             let index_of_parent = identity_to_json_map[parent_identity_post];
-            // let mut updated = JSONEntry::from_field(sorted_entries[index_of_parent]);
-            // updated.child_pointer = i as Field + 1;
+            let mut updated = JSONEntry::from_field(sorted_entries[index_of_parent]);
+            updated.child_pointer = i as Field + 1;
 
             // // RELIES ON THE SMALLEST ENTRY IN THE SORTED LIST BEING EMPTY
-            // let index = ((index_of_parent - 0) * new_parent as Field) + 0;
-
-            // sorted_entries[index] = updated.to_field();
-            if (new_parent) {
-                let index_of_parent = identity_to_json_map[parent_identity_post];
-                let mut updated = JSONEntry::from_field(sorted_entries[index_of_parent]);
-                updated.child_pointer = i as Field + 1;
-                sorted_entries[index_of_parent] = updated.to_field();
-            }
+            let index = ((index_of_parent - 0) * new_parent as Field) + 0;
+            sorted_entries[index] = updated.to_field();
+            // if (new_parent) {
+            //     let index_of_parent = identity_to_json_map[parent_identity_post];
+            //     let mut updated = JSONEntry::from_field(sorted_entries[index_of_parent]);
+            //     updated.child_pointer = i as Field + 1;
+            //     sorted_entries[index_of_parent] = updated.to_field();
+            // }
             // i + 1 is the starting index of a new set of children
         }
 
@@ -164,6 +178,9 @@ impl<let NumBytes: u32, let NumPackedFields: u16, let TranscriptEntries: u32> JS
         self.packed_json_entries = sorted_entries;
 
         self.key_hashes = sort_result.sorted;
+        // 38050
+        // 60707 <-- cost after hashing keuys
+        // 77244 <-- cost after sorting and updating children
     }
 }
 

src/keymap.nr

@@ -612,7 +612,8 @@ fn mainkjghg(body_text: [u8; 1024], body_indices: [u16; 64], key_lengths: [u16;
 
 // ok new
 
-// build_transcript: 23375
+// empty: 1332
+// build_transcript: 21332
 // + capture_missing_tokens: 24233
 // + keyswap: 33605 (9k?)
 // + create_json_entries: 53989 (20k?)
@@ -691,15 +692,16 @@ fn main(text: str<1024>) {
     json.keyswap();
     json.compute_packed_json();
     json.create_json_entries();
-    // assert(json.packed_json[inputs[3]] == 1234);
-
-    for i in 0..63 {
-        assert(json.key_data[i] == json.key_data[i + 1]);
-    }
+    // // assert(json.packed_json[inputs[3]] == 1234);
+    // for i in 0..63 {
+    //     assert(json.key_data[i] == json.key_data[i + 1]);
+    // }
     //  let keymap::KeyIndexData{ json_index, json_length, parent_id, array_index } = keymap::KeyIndexData::from_field(json.key_data[0]);
     // assert(json_index == json_length);
     // assert(parent_id == array_index);
-    // json.compute_keyhash_and_sort_json_entries();
+
+    // 62578
+    json.compute_keyhash_and_sort_json_entries();
     // 51810 57908
     // 51889 57911
     // delta of 6,098 even after initing range tables

src/main.nr

@@ -603,7 +603,7 @@ impl<let NumBytes: u16, let NumPackedFields: u16, let TranscriptEntries: u16> JS
             let object_or_array_entry: JSONEntry = JSONEntry {
                     array_pointer: previous_stack_entry.num_entries, // duplicated lookup remove once working 
                     entry_type: json_entry_type,
-                    child_pointer: previous_stack_entry.entry_pointer, // need a stack to figure this out. is depth value correct here?
+                    child_pointer: 0,// previous_stack_entry.entry_pointer, // need a stack to figure this out. is depth value correct here?
                     num_children: num_entries_at_current_depth, // no children
                     json_pointer: previous_stack_entry.json_index,
                     json_length: length,
@@ -726,32 +726,61 @@ impl<let NumBytes: u16, let NumPackedFields: u16, let TranscriptEntries: u16> JS
             // 1 gate to compute index
             // 2 gates to read
             let capture_context = CAPTURE_TABLE_ENCODED_FLAT[scan_mode * 128 + ascii as Field];
-            // 2 gates for 5 bytes + 1.25 range gates = 3.25
+            // 2 gates for 5 bytes + 5 range gates = 8.25
             let bytes = capture_context.to_be_bytes(5);
             let new_scan_mode = bytes[4] as Field;
             let scan_token = bytes[3] as Field;
             let push_transcript = bytes[2] as Field;
             let increase_length = bytes[1] as Field;
             let error_flag = bytes[0] as Field;
 
+            // what if we store the following:
+            // 1 ascii
+            // 2 scan mode
+            // 3 push_transcript
+            // 4 error flag (ick)
+            // but we avoid scan token, we use a future lookup for that
+
+            // ascii
+            // scan mode
+            // push transcript
+            // update length
+            // convert bytes would cost 4.75 instead of 8.25?
+            // 3.5 gate saving
+            // + remove error flag check = 1 gate saving
+
+            // + can fiddle with length for 1 gate saving
+            // 5.5 gate delta on 19.75
+            // -> 14.25 gates per byte
+            // could cut 0.75 by doing dumb bool checks
+            // = 6.25 delta
+            // use ROM array instead of RAM = 0.5
+            // 6.75 gate delta
+            // 13 per byte
             // 1 gate for i - length
             // 1 gate for to_field
-            // subtotal: 8.25
+            // subtotal: 13.25
             let new_entry = TranscriptEntry::to_field(TranscriptEntry { token: scan_token, index: i as Field - length, length });
 
             // let old_entry = transcript[transcript_ptr];
 
             // let entry = (new_entry - old_entry) * push_transcript + old_entry;
             // 3.5 gates to write
             // TODO might be a problem this last token
+            // 16.75
             transcript[transcript_ptr] = new_entry; //  * push_transcript;
             // 1 gate to update length
             length = length * (1 - push_transcript) + increase_length;
+            std::as_witness(length);
             // 1 gate to update transcript_ptr
-            // 13.75
+            // 18.75
             transcript_ptr += push_transcript;
 
             // hmm should be 0 gates but might be 1
+            // 19.75
+            // actual cost = 20
+            // missing a gate from transcript entry length param?
+            // odd maybe 1.25 not being used
             assert(error_flag == 0, "bad token?");
 
             scan_mode = new_scan_mode;
@@ -1050,7 +1079,7 @@ struct JSONEntry {
         json_entries[3] == JSONEntry {
             array_pointer: 1,
             entry_type: BEGIN_OBJECT_TOKEN,
-            child_pointer: 1, // first child of object is json entry 1
+            child_pointer: 0, // first child of object is json entry 1
             num_children: 2,
             json_pointer: get(7).index,
             json_length: get(7).length,