Merge branch 'main' into jl/documentation

Author: TomAFrench

src/_string_tools/slice_packed_field.nr

@@ -756,7 +756,7 @@ pub fn slice_fields<let InputFields: u32, let OutputFields: u32>(
     let use_previous_for_last_limb: Field = extra_head_section + bytes_fit_into_limb;
 
     // 1, 132.5
-    let mut index_of_overflow_limb = start_index + num_whole_limbs + 1;
+    let index_of_overflow_limb = start_index + num_whole_limbs + 1;
     // 2, 134.5
     let last_limb_from_data = data[index_of_overflow_limb as u32];
     // 2, 136.5

src/_table_generation/make_tables.nr

@@ -288,8 +288,8 @@ unconstrained fn make_token_validation_table() -> [Field; NUM_TOKENS * NUM_TOKEN
     let mut flattened_flags: [Field; NUM_TOKENS * NUM_TOKENS * 3] =
         [0; NUM_TOKENS * NUM_TOKENS * 3];
     let NN = (NUM_TOKENS * NUM_TOKENS);
-    for j in 0..NUM_TOKENS as u32 {
-        for k in 0..NUM_TOKENS as u32 {
+    for j in 0..NUM_TOKENS {
+        for k in 0..NUM_TOKENS {
             flattened_flags[OBJECT_LAYER * NN + j * NUM_TOKENS + k] = object_layer_flags[j][k];
             flattened_flags[ARRAY_LAYER * NN + j * NUM_TOKENS + k] = array_layer_flags[j][k];
             flattened_flags[SINGLE_VALUE_LAYER * NN + j * NUM_TOKENS + k] =
@@ -359,94 +359,94 @@ unconstrained fn make_process_raw_transcript_table() -> [Field; 1024] {
 }
 
 unconstrained fn generate_token_flags_table() -> [Field; NUM_TOKENS * 2] {
-    let mut flags: [TokenFlags; NUM_TOKENS * 2] = [TokenFlags::default(); NUM_TOKENS * 2];
+    let mut flags: [TokenFlags; NUM_TOKENS * 2] = std::mem::zeroed();
 
     let mut no_token_flags: TokenFlags = TokenFlags {
         create_json_entry: false,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 0,
-        new_context: OBJECT_LAYER as Field,
-        is_key_token: 0,
-        is_value_token: 0,
-        preserve_num_entries: 1,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: false,
+        new_context: OBJECT_LAYER != 0,
+        is_key_token: false,
+        is_value_token: false,
+        preserve_num_entries: true,
     };
     let mut key_token_flags: TokenFlags = TokenFlags {
         create_json_entry: false,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 0,
-        new_context: OBJECT_LAYER as Field,
-        is_key_token: 1,
-        is_value_token: 0,
-        preserve_num_entries: 1,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: false,
+        new_context: OBJECT_LAYER != 0,
+        is_key_token: true,
+        is_value_token: false,
+        preserve_num_entries: true,
     };
     let begin_object_flags = TokenFlags {
         create_json_entry: false,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 1,
-        new_context: OBJECT_LAYER as Field,
-        is_key_token: 0,
-        is_value_token: 0,
-        preserve_num_entries: 0,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: true,
+        new_context: OBJECT_LAYER != 0,
+        is_key_token: false,
+        is_value_token: false,
+        preserve_num_entries: false,
     };
 
     let begin_array_flags = TokenFlags {
         create_json_entry: false,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 1,
-        new_context: ARRAY_LAYER as Field,
-        is_key_token: 0,
-        is_value_token: 0,
-        preserve_num_entries: 0,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: true,
+        new_context: ARRAY_LAYER != 0,
+        is_key_token: false,
+        is_value_token: false,
+        preserve_num_entries: false,
     };
 
     let mut end_object_flags = TokenFlags {
         create_json_entry: true,
-        is_end_of_object_or_array: 1,
-        is_start_of_object_or_array: 0,
-        new_context: 0,
-        is_key_token: 0,
-        is_value_token: 0,
-        preserve_num_entries: 0,
+        is_end_of_object_or_array: true,
+        is_start_of_object_or_array: false,
+        new_context: false,
+        is_key_token: false,
+        is_value_token: false,
+        preserve_num_entries: false,
     };
 
     let mut end_array_flags = TokenFlags {
         create_json_entry: true,
-        is_end_of_object_or_array: 1,
-        is_start_of_object_or_array: 0,
-        new_context: 0,
-        is_key_token: 0,
-        is_value_token: 0,
-        preserve_num_entries: 0,
+        is_end_of_object_or_array: true,
+        is_start_of_object_or_array: false,
+        new_context: false,
+        is_key_token: false,
+        is_value_token: false,
+        preserve_num_entries: false,
     };
 
     let mut string_flags = TokenFlags {
         create_json_entry: true,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 0,
-        new_context: OBJECT_LAYER as Field,
-        is_key_token: 0,
-        is_value_token: 1,
-        preserve_num_entries: 1,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: false,
+        new_context: OBJECT_LAYER != 0,
+        is_key_token: false,
+        is_value_token: true,
+        preserve_num_entries: true,
     };
 
     let mut numeric_flags = TokenFlags {
         create_json_entry: true,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 0,
-        new_context: OBJECT_LAYER as Field,
-        is_key_token: 0,
-        is_value_token: 1,
-        preserve_num_entries: 1,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: false,
+        new_context: OBJECT_LAYER != 0,
+        is_key_token: false,
+        is_value_token: true,
+        preserve_num_entries: true,
     };
 
     let mut literal_flags = TokenFlags {
         create_json_entry: true,
-        is_end_of_object_or_array: 0,
-        is_start_of_object_or_array: 0,
-        new_context: OBJECT_LAYER as Field,
-        is_key_token: 0,
-        is_value_token: 1,
-        preserve_num_entries: 1,
+        is_end_of_object_or_array: false,
+        is_start_of_object_or_array: false,
+        new_context: OBJECT_LAYER != 0,
+        is_key_token: false,
+        is_value_token: true,
+        preserve_num_entries: true,
     };
 
     flags[NO_TOKEN] = no_token_flags;
@@ -461,26 +461,26 @@ unconstrained fn generate_token_flags_table() -> [Field; NUM_TOKENS * 2] {
     flags[LITERAL_TOKEN] = literal_flags;
     flags[KEY_TOKEN] = key_token_flags;
 
-    no_token_flags.new_context = ARRAY_LAYER as Field;
-    key_token_flags.new_context = ARRAY_LAYER as Field;
-    string_flags.new_context = ARRAY_LAYER as Field;
-    numeric_flags.new_context = ARRAY_LAYER as Field;
-    literal_flags.new_context = ARRAY_LAYER as Field;
-
-    flags[NUM_TOKENS + (NO_TOKEN as u32)] = no_token_flags;
-    flags[NUM_TOKENS + (BEGIN_OBJECT_TOKEN as u32)] = begin_object_flags;
-    flags[NUM_TOKENS + (END_OBJECT_TOKEN as u32)] = end_object_flags;
-    flags[NUM_TOKENS + (BEGIN_ARRAY_TOKEN as u32)] = begin_array_flags;
-    flags[NUM_TOKENS + (END_ARRAY_TOKEN as u32)] = end_array_flags;
-    flags[NUM_TOKENS + (KEY_SEPARATOR_TOKEN as u32)] = no_token_flags;
-    flags[NUM_TOKENS + (VALUE_SEPARATOR_TOKEN as u32)] = no_token_flags;
-    flags[NUM_TOKENS + (STRING_TOKEN as u32)] = string_flags;
-    flags[NUM_TOKENS + (NUMERIC_TOKEN as u32)] = numeric_flags;
-    flags[NUM_TOKENS + (LITERAL_TOKEN as u32)] = literal_flags;
-    flags[NUM_TOKENS + (KEY_TOKEN as u32)] = key_token_flags;
+    no_token_flags.new_context = ARRAY_LAYER != 0;
+    key_token_flags.new_context = ARRAY_LAYER != 0;
+    string_flags.new_context = ARRAY_LAYER != 0;
+    numeric_flags.new_context = ARRAY_LAYER != 0;
+    literal_flags.new_context = ARRAY_LAYER != 0;
+
+    flags[NUM_TOKENS + NO_TOKEN] = no_token_flags;
+    flags[NUM_TOKENS + BEGIN_OBJECT_TOKEN] = begin_object_flags;
+    flags[NUM_TOKENS + END_OBJECT_TOKEN] = end_object_flags;
+    flags[NUM_TOKENS + BEGIN_ARRAY_TOKEN] = begin_array_flags;
+    flags[NUM_TOKENS + END_ARRAY_TOKEN] = end_array_flags;
+    flags[NUM_TOKENS + KEY_SEPARATOR_TOKEN] = no_token_flags;
+    flags[NUM_TOKENS + VALUE_SEPARATOR_TOKEN] = no_token_flags;
+    flags[NUM_TOKENS + STRING_TOKEN] = string_flags;
+    flags[NUM_TOKENS + NUMERIC_TOKEN] = numeric_flags;
+    flags[NUM_TOKENS + LITERAL_TOKEN] = literal_flags;
+    flags[NUM_TOKENS + KEY_TOKEN] = key_token_flags;
 
     let mut result: [Field; NUM_TOKENS * 2] = [0; NUM_TOKENS * 2];
-    for i in 0..(NUM_TOKENS as u32 * 2) {
+    for i in 0..(NUM_TOKENS * 2) {
         result[i] = flags[i].to_field();
     }
     result

src/getters.nr

@@ -94,7 +94,7 @@ impl<let NumBytes: u32, let NumPackedFields: u32, let MaxNumTokens: u32, let Max
         assert(self.layer_type_of_root != ARRAY_LAYER, "cannot extract array elements via a key");
 
         let hasher: ByteHasher<MaxKeyFields> = ByteHasher {};
-        let keyhash = hasher.get_keyhash_var(key.storage(), 0, key.len() as u32);
+        let keyhash = hasher.get_keyhash_var(key.storage(), 0, key.len());
         let two_pow_216 = 0x100000000000000000000000000000000000000000000000000000000;
 
         let keyhash = keyhash + self.root_id * two_pow_216;
@@ -147,7 +147,7 @@ impl<let NumBytes: u32, let NumPackedFields: u32, let MaxNumTokens: u32, let Max
         assert(self.layer_type_of_root != ARRAY_LAYER, "cannot extract array elements via a key");
 
         let hasher: ByteHasher<MaxKeyFields> = ByteHasher {};
-        let keyhash = hasher.get_keyhash_var(key.storage(), 0, key.len() as u32);
+        let keyhash = hasher.get_keyhash_var(key.storage(), 0, key.len());
         let two_pow_216 = 0x100000000000000000000000000000000000000000000000000000000;
 
         let keyhash = keyhash + self.root_id * two_pow_216;
@@ -395,7 +395,7 @@ impl<let NumBytes: u32, let NumPackedFields: u32, let MaxNumTokens: u32, let Max
             case 3: entry A > keyhash > entryB
         */
         let hasher: ByteHasher<MaxKeyFields> = ByteHasher {};
-        let keyhash = hasher.get_keyhash_var(key.storage(), 0, key.len() as u32);
+        let keyhash = hasher.get_keyhash_var(key.storage(), 0, key.len());
 
         let HASH_MAXIMUM = 0x1000000000000000000000000000000000000000000000000000000000000 - 1;
         let two_pow_216 = 0x100000000000000000000000000000000000000000000000000000000;

src/json.nr

@@ -321,10 +321,20 @@ impl<let NumBytes: u32, let NumPackedFields: u32, let MaxNumTokens: u32, let Max
                 TOKEN_FLAGS_TABLE[cast_num_to_u32(token) + context * NUM_TOKENS],
             );
 
+            // We convert these booleans into Fields so that we can use them in arithmetic operations.
+            let create_json_entry = create_json_entry as Field;
+            let is_end_of_object_or_array = is_end_of_object_or_array as Field;
+            let is_start_of_object_or_array = is_start_of_object_or_array as Field;
+            let new_context = new_context as Field;
+            let update_key = update_key as Field;
+            let is_value_token = is_value_token as Field;
+            let preserve_num_entries = preserve_num_entries as Field;
+
             // Determine what the current key index is and the key size in bytes
             // (key index = byte location in the original JSON of the key)
             // Pseudocode equivalent:
             // current_key_index_and_length = update_key ? (index + length * 0x10000) : current_key_index_and_length
+
             // 2 gates
             // only update current_key_index_and_length if the token is a key token
             let diff = (index + length * 0x10000) - current_key_index_and_length;

src/keyhash.nr

@@ -18,7 +18,7 @@ impl<let KeyFields: u32> FieldHasher<KeyFields> {
         key_length: Field,
     ) -> Field {
         let key_fields: [Field; KeyFields] = slice_fields(packed_fields, body_index, key_length);
-        let hashed = poseidon::poseidon2::Poseidon2::hash(key_fields, KeyFields as u32);
+        let hashed = poseidon::poseidon2::Poseidon2::hash(key_fields, KeyFields);
         slice_200_bits_from_field(hashed)
     }
 }
@@ -60,7 +60,7 @@ impl<let KeyFields: u32> ByteHasher<KeyFields> {
             key_fields[j] = limb;
         }
 
-        let hashed_full = poseidon::poseidon2::Poseidon2::hash(key_fields, KeyFields as u32);
+        let hashed_full = poseidon::poseidon2::Poseidon2::hash(key_fields, KeyFields);
 
         let mut r = slice_200_bits_from_field(hashed_full);
         r

src/token_flags.nr

@@ -14,12 +14,12 @@
 ///     - when `preserve_num_entries == 1` implies the current parent object has not changed (could do with renaming this variable)
 pub(crate) struct TokenFlags {
     pub(crate) create_json_entry: bool,
-    pub(crate) is_end_of_object_or_array: Field,
-    pub(crate) is_start_of_object_or_array: Field,
-    pub(crate) new_context: Field,
-    pub(crate) is_key_token: Field,
-    pub(crate) is_value_token: Field,
-    pub(crate) preserve_num_entries: Field,
+    pub(crate) is_end_of_object_or_array: bool,
+    pub(crate) is_start_of_object_or_array: bool,
+    pub(crate) new_context: bool,
+    pub(crate) is_key_token: bool,
+    pub(crate) is_value_token: bool,
+    pub(crate) preserve_num_entries: bool,
 }
 
 impl TokenFlags {
@@ -33,12 +33,12 @@ impl TokenFlags {
     unconstrained fn __from_field(f: Field) -> Self {
         let bytes: [u8; 7] = f.to_be_bytes();
         let create_json_entry = bytes[0] != 0;
-        let is_end_of_object_or_array = bytes[1] as Field;
-        let is_start_of_object_or_array = bytes[2] as Field;
-        let new_context = bytes[3] as Field;
-        let is_key_token = bytes[4] as Field;
-        let is_value_token = bytes[5] as Field;
-        let preserve_num_entries = bytes[6] as Field;
+        let is_end_of_object_or_array = bytes[1] != 0;
+        let is_start_of_object_or_array = bytes[2] != 0;
+        let new_context = bytes[3] != 0;
+        let is_key_token = bytes[4] != 0;
+        let is_value_token = bytes[5] != 0;
+        let preserve_num_entries = bytes[6] != 0;
 
         TokenFlags {
             create_json_entry,
@@ -56,21 +56,6 @@ impl TokenFlags {
         // 10 gates
         // Safety: check the comments below
         let r = unsafe { TokenFlags::__from_field(f) };
-
-        // checks that the flags are binary
-        assert(
-            r.is_end_of_object_or_array * r.is_end_of_object_or_array
-                == r.is_end_of_object_or_array,
-        );
-        assert(
-            r.is_start_of_object_or_array * r.is_start_of_object_or_array
-                == r.is_start_of_object_or_array,
-        );
-        assert(r.new_context * r.new_context == r.new_context);
-        assert(r.is_key_token * r.is_key_token == r.is_key_token);
-        assert(r.is_value_token * r.is_value_token == r.is_value_token);
-        assert(r.preserve_num_entries * r.preserve_num_entries == r.preserve_num_entries);
-
         // asserts the relation of r and f
         assert(r.to_field() == f);
         r
@@ -79,25 +64,12 @@ impl TokenFlags {
     /// Pack a TokenFlags object into a Field element
     /// 4 gates
     pub(crate) fn to_field(self) -> Field {
-        self.preserve_num_entries
-            + self.is_value_token * 0x100
-            + self.is_key_token * 0x10000
-            + self.new_context * 0x1000000
-            + self.is_start_of_object_or_array * 0x100000000
-            + self.is_end_of_object_or_array * 0x10000000000
-            + self.create_json_entry as Field * 0x1000000000000
-    }
-
-    /// Default constructor
-    pub(crate) fn default() -> Self {
-        TokenFlags {
-            create_json_entry: false,
-            is_end_of_object_or_array: 0,
-            is_start_of_object_or_array: 0,
-            new_context: 0,
-            is_key_token: 0,
-            preserve_num_entries: 0,
-            is_value_token: 0,
-        }
+        (self.preserve_num_entries as Field)
+            + (self.is_value_token as Field) * 0x100
+            + (self.is_key_token as Field) * 0x10000
+            + (self.new_context as Field) * 0x1000000
+            + (self.is_start_of_object_or_array as Field) * 0x100000000
+            + (self.is_end_of_object_or_array as Field) * 0x10000000000
+            + (self.create_json_entry as Field) * 0x1000000000000
     }
 }