Merge branch 'compute-size-and-assert-valid-size-indicator'

Add a derivable function to the `TasmObject`:
`compute_size_and_assert_valid_size_indicator`. This function returns
code that returns the size of the BFieldCodec encoded struct in memory.
Furthermore, all size indicators in the structure are verified. This
function is intended to be called on pre-loaded objects that already
exist in memory, as the program execution starts.

The functionality comes neatly wrapped in a snippet: `VerifyNdSiIntegrity`.

Author: Sword-Smith

derive_tasm_object/Cargo.toml

@@ -7,6 +7,7 @@ description = "Derive macro for TasmObject"
 [dependencies]
 quote = "1.0"
 syn = { version = "^2", features = ["full", "derive"] }
+proc-macro2 = { version = "1" }
 
 [lib]
 proc-macro = true

derive_tasm_object/src/lib.rs

@@ -53,6 +53,129 @@ struct ParseResult {
     ignored_fields: Vec<syn::Field>,
 }
 
+fn generate_integral_size_indicators_code(parse_result: &ParseResult) -> proc_macro2::TokenStream {
+    let mut integral_size_indicators_code = quote! {};
+    for field_type in parse_result.field_types.iter() {
+        // INVARIANT: accumulated_size *field_start
+        integral_size_indicators_code = quote! {
+            #integral_size_indicators_code
+            // _ accumulated_size *field_start
+            if let Some(static_length) = <#field_type as crate::triton_vm::twenty_first::math::bfield_codec::BFieldCodec>::static_length() {
+                let addi_len = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::AddI(crate::BFieldElement::new(static_length as u64)));
+                [
+                    // _ accumulated_size *field
+                    addi_len.clone(),
+                    swap1.clone(),
+                    addi_len.clone(),
+                    swap1.clone(),
+                    // _ accumulated_size' *next_field
+                ].to_vec()
+            } else {
+                [
+                    [
+                        // _ accumulated_size *indicated_field_size
+                        read_mem1.clone(),
+                        // _ accumulated_size indicated_field_size (*field-2)
+
+                        pushmax.clone(),
+                        dup2.clone(),
+                        lt.clone(),
+                        // _ accumulated_size indicated_field_size (*field-2) (MAX > indicated_field_size)
+
+                        assert.clone(),
+                        // _ accumulated_size indicated_field_size (*field-2)
+
+                        addi2.clone(),
+                        // _ accumulated_size indicated_field_size *field
+
+                        dup0.clone(),
+                        // _ accumulated_size indicated_field_size *field *field
+                    ].to_vec(),
+                    <#field_type as crate::tasm_lib::structure::tasm_object::TasmObject>::compute_size_and_assert_valid_size_indicator(library),
+                    // _ accumulated_size indicated_field_size *field computed_size
+                    [
+                        dup2.clone(),
+                        // _ accumulated_size indicated_field_size *field computed_size indicated_field_size
+                        eq.clone(),
+                        // _ accumulated_size indicated_field_size *field (computed_size == indicated_field_size)
+                        assert.clone(),
+                        // _ accumulated_size indicated_field_size *field
+
+                        dup1.clone(),
+                        add.clone(),
+                        // _ accumulated_size indicated_field_size *next_field
+
+                        swap2.clone(),
+                        // _ *next_field indicated_field_size accumulated_size
+
+                        /* Add one for size-indicator on this struct */
+                        add.clone(),
+                        addi1.clone(),
+                        // _ *next_field accumulated_size'
+
+                        swap1.clone(),
+                        // _ accumulated_size' *next_field
+                    ].to_vec(),
+                ].concat()
+            },
+        };
+    }
+
+    integral_size_indicators_code = quote! {
+        let push0 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Push(0u64.into()));
+        let pushmax = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Push(Self::MAX_OFFSET.into()));
+        let dup0 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Dup(crate::triton_vm::op_stack::OpStackElement::ST0));
+        let dup1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Dup(crate::triton_vm::op_stack::OpStackElement::ST1));
+        let dup2 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Dup(crate::triton_vm::op_stack::OpStackElement::ST2));
+        let swap1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Swap(crate::triton_vm::op_stack::OpStackElement::ST1));
+        let swap2 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Swap(crate::triton_vm::op_stack::OpStackElement::ST2));
+        let lt = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Lt);
+        let assert = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Assert);
+        let eq = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Eq);
+        let add = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Add);
+        let read_mem1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::ReadMem(crate::triton_vm::op_stack::NumberOfWords::N1));
+        let addi1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::AddI(crate::BFieldElement::new(1u64)));
+        let addi2 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::AddI(crate::BFieldElement::new(2u64)));
+        let pop1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Pop(crate::triton_vm::op_stack::NumberOfWords::N1));
+        let hint_acc_size = [
+                    crate::triton_vm::instruction::LabelledInstruction::TypeHint(
+                        crate::triton_vm::instruction::TypeHint {
+                            starting_index: 1,
+                            length: 1,
+                            type_name: std::option::Option::<std::string::String>::None,
+                            variable_name: std::string::String::from("acc_size"),
+                        }
+                    )
+                ].to_vec();
+        let hint_field_ptr = [
+            crate::triton_vm::instruction::LabelledInstruction::TypeHint(
+                crate::triton_vm::instruction::TypeHint {
+                    starting_index: 0,
+                    length: 1,
+                    type_name: std::option::Option::<std::string::String>::None,
+                    variable_name: std::string::String::from("field_ptr"),
+                }
+            )
+        ].to_vec();
+        [
+            [
+                push0.clone(),
+                swap1.clone(),
+            ].to_vec(),
+            hint_acc_size,
+            hint_field_ptr,
+            #integral_size_indicators_code
+            [
+                // _ acc_size *EOF
+                pop1.clone(),
+                // _ acc_size
+            ].to_vec(),
+        ].concat()
+    };
+
+    integral_size_indicators_code
+}
+
 fn impl_derive_tasm_object_macro(ast: DeriveInput) -> TokenStream {
     let parse_result = generate_parse_result(&ast);
 
@@ -181,50 +304,17 @@ fn impl_derive_tasm_object_macro(ast: DeriveInput) -> TokenStream {
         }
     };
 
-    let encoding_length = if let Some(tokens) = get_current_field_start_with_jump.clone().last() {
-        quote! {
-            let first_field_with_jump_distance = { #tokens };
-            let add = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Add);
-            let dup0 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Dup(crate::triton_vm::op_stack::OpStackElement::ST0));
-            let swap1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Swap(crate::triton_vm::op_stack::OpStackElement::ST1));
-            let push_neg_1 = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Push(crate::triton_vm::prelude::BFieldElement::new(crate::triton_vm::prelude::BFieldElement::P-1)));
-            let mul = crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Mul);
-
-            // Get `*field` and `size` for last field of encoding, which is the 1st in the definition.
-            // Then add them to a pointer one past the end of the object. Then subtract `*object`
-            // of the object to get final size.
-            let extract_encoding_size = [
-                    // _ *object *field size
-                    add.clone(),
-                    // _ *object (*field+size)
-                    swap1,
-                    // _ (*field+size) *object
-                    push_neg_1,
-                    // _ (*field+size) *object -1
-                    mul,
-                    // _ (*field+size) (-*object)
-                    add
-                    // _ encoding_size
-                ].to_vec();
-
-            [
-                [dup0].to_vec(),
-                first_field_with_jump_distance,
-                extract_encoding_size].concat()
-        }
-    } else {
-        quote! {
-            [
-                crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Pop(crate::triton_vm::op_stack::NumberOfWords::N1)),
-                crate::triton_vm::instruction::LabelledInstruction::Instruction(crate::triton_vm::instruction::AnInstruction::Push(crate::triton_vm::prelude::BFieldElement::new(0u64))),
-            ].to_vec()
-        }
-    };
+    let integral_size_indicators_code = generate_integral_size_indicators_code(&parse_result);
 
     let name = &ast.ident;
+    let name_as_string = ast.ident.to_string();
     let gen = quote! {
         impl #impl_generics crate::tasm_lib::structure::tasm_object::TasmObject
         for #name #ty_generics #new_where_clause {
+            fn label_friendly_name() -> String {
+                #name_as_string.to_owned()
+            }
+
             fn get_field( field_name : &str ) -> Vec<crate::triton_vm::instruction::LabelledInstruction> {
                 let field_getter = match field_name {
                     #( #just_field_clauses ,)*
@@ -283,8 +373,8 @@ fn impl_derive_tasm_object_macro(ast: DeriveInput) -> TokenStream {
                 }
             }
 
-            fn get_encoding_length() -> Vec<crate::triton_vm::instruction::LabelledInstruction> {
-                #encoding_length
+            fn compute_size_and_assert_valid_size_indicator(library: &mut crate::tasm_lib::Library) -> Vec<crate::triton_vm::instruction::LabelledInstruction> {
+                #integral_size_indicators_code
             }
 
             fn decode_iter<Itr: Iterator<Item=crate::BFieldElement>>(

tasm-lib/benchmarks/tasmlib_structure_verify_nd_si_integrity___ProofCollectionLookalike.json

@@ -0,0 +1,24 @@
+[
+  {
+    "name": "tasmlib_structure_verify_nd_si_integrity___ProofCollectionLookalike",
+    "benchmark_result": {
+      "clock_cycle_count": 205,
+      "hash_table_height": 234,
+      "u32_table_height": 150,
+      "op_stack_table_height": 136,
+      "ram_table_height": 16
+    },
+    "case": "CommonCase"
+  },
+  {
+    "name": "tasmlib_structure_verify_nd_si_integrity___ProofCollectionLookalike",
+    "benchmark_result": {
+      "clock_cycle_count": 289,
+      "hash_table_height": 234,
+      "u32_table_height": 212,
+      "op_stack_table_height": 196,
+      "ram_table_height": 22
+    },
+    "case": "WorstCase"
+  }
+]

tasm-lib/benchmarks/tasmlib_structure_verify_nd_si_integrity___TransactionKernelLookalike.json

@@ -0,0 +1,24 @@
+[
+  {
+    "name": "tasmlib_structure_verify_nd_si_integrity___TransactionKernelLookalike",
+    "benchmark_result": {
+      "clock_cycle_count": 270,
+      "hash_table_height": 384,
+      "u32_table_height": 240,
+      "op_stack_table_height": 176,
+      "ram_table_height": 15
+    },
+    "case": "CommonCase"
+  },
+  {
+    "name": "tasmlib_structure_verify_nd_si_integrity___TransactionKernelLookalike",
+    "benchmark_result": {
+      "clock_cycle_count": 236,
+      "hash_table_height": 384,
+      "u32_table_height": 212,
+      "op_stack_table_height": 156,
+      "ram_table_height": 14
+    },
+    "case": "WorstCase"
+  }
+]

tasm-lib/src/hashing/algebraic_hasher/sample_scalars_static_length_kmalloc.rs

@@ -33,12 +33,6 @@ impl SampleScalarsStaticLengthKMalloc {
             .try_into()
             .unwrap()
     }
-
-    pub(crate) fn scalars_kmalloc_name(&self) -> String {
-        let num_elements_to_sample = self.num_elements_to_sample;
-        let extra_capacity = self.extra_capacity;
-        format!("scalars_kmalloc_{num_elements_to_sample}_{extra_capacity}")
-    }
 }
 
 impl BasicSnippet for SampleScalarsStaticLengthKMalloc {
@@ -60,6 +54,10 @@ impl BasicSnippet for SampleScalarsStaticLengthKMalloc {
     fn code(&self, library: &mut Library) -> Vec<LabelledInstruction> {
         assert_eq!(10, tip5::RATE, "Code assumes Tip5's RATE is 10");
         assert_eq!(3, EXTENSION_DEGREE, "Code assumes extension degree 3");
+        assert!(
+            self.extra_capacity + self.num_elements_to_sample > 0,
+            "Must allocate positive number of words"
+        );
         let num_squeezes =
             SampleScalarsStaticLengthDynMalloc::num_squeezes(self.num_elements_to_sample);
 
@@ -73,8 +71,7 @@ impl BasicSnippet for SampleScalarsStaticLengthKMalloc {
         let entrypoint = self.entrypoint();
         let squeeze_repeatedly_static_number =
             library.import(Box::new(SqueezeRepeatedlyStaticNumber { num_squeezes }));
-        let scalars_pointer =
-            library.pub_kmalloc(self.num_words_to_allocate(), self.scalars_kmalloc_name());
+        let scalars_pointer = library.kmalloc(self.num_words_to_allocate());
 
         triton_asm!(
             {entrypoint}:
@@ -182,8 +179,8 @@ pub(crate) mod tests {
 
     #[proptest]
     fn verify_agreement_with_tip5_sample_scalars(
-        #[strategy(0_usize..500)] num_elements_to_sample: usize,
-        #[strategy(0_usize..500)] extra_capacity: usize,
+        #[strategy(1_usize..500)] num_elements_to_sample: usize,
+        #[strategy(1_usize..500)] extra_capacity: usize,
         #[strategy(arb())] mut sponge: Tip5,
     ) {
         let snippet = SampleScalarsStaticLengthKMalloc {

tasm-lib/src/library.rs

@@ -29,10 +29,6 @@ pub struct Library {
     /// Imported dependencies.
     seen_snippets: HashMap<String, Vec<LabelledInstruction>>,
 
-    /// Known, and thus shareable, (static) memory allocations. Includes both their address and
-    /// their size.
-    pub_allocations: HashMap<String, (BFieldElement, u32)>,
-
     /// The number of statically allocated words
     num_allocated_words: u32,
 }
@@ -51,7 +47,6 @@ impl Library {
     pub fn new() -> Self {
         Self {
             seen_snippets: HashMap::default(),
-            pub_allocations: HashMap::default(),
             num_allocated_words: 0,
         }
     }
@@ -89,6 +84,13 @@ impl Library {
         dep_entrypoint
     }
 
+    /// Import code that does not implement the `Snippet` trait
+    ///
+    /// If possible, you should use the [`import`](Self::import) method as
+    /// it gives better protections and allows you to test functions in
+    /// isolation. This method is intended to add function to the assembly
+    /// that you have defined inline and where a function call is needed due to
+    /// e.g. a dynamic counter.
     pub fn explicit_import(&mut self, name: &str, body: &[LabelledInstruction]) -> String {
         if !self.seen_snippets.contains_key(name) {
             self.seen_snippets.insert(name.to_owned(), body.to_vec());
@@ -123,6 +125,7 @@ impl Library {
     /// Statically allocate `num_words` words of memory. Panics if more static
     /// memory is required than what the capacity allows for.
     pub fn kmalloc(&mut self, num_words: u32) -> BFieldElement {
+        assert!(num_words > 0, "must allocate a positive number of words");
         let address = STATIC_MEMORY_FIRST_ADDRESS
             - bfe!(self.num_allocated_words)
             - BFieldElement::new(num_words as u64 - 1);
@@ -133,24 +136,6 @@ impl Library {
 
         address
     }
-
-    /// Statically allocate `num_words` words of memory and give it a name.
-    /// Allows sharing the allocation with other snippets.
-    pub fn pub_kmalloc(&mut self, num_words: u32, name: String) -> BFieldElement {
-        let address = self.kmalloc(num_words);
-        if let Some((addr, size)) = self
-            .pub_allocations
-            .insert(name.clone(), (address, num_words))
-        {
-            panic!("Public kmalloc for \"{name}\" overwrote previous allocation: ({addr}, {size})");
-        };
-        address
-    }
-
-    /// Get the address and size of a public allocation.
-    pub fn get_pub_allocation(&self, name: &str) -> (BFieldElement, u32) {
-        self.pub_allocations[name]
-    }
 }
 
 #[derive(Debug)]
@@ -489,12 +474,5 @@ mod tests {
 
         let third_free_address = lib.kmalloc(1000);
         assert_eq!(-BFieldElement::new(1009), third_free_address);
-
-        let fourth_free_address = lib.pub_kmalloc(10_000, "my_thing".to_string());
-        assert_eq!(-BFieldElement::new(11_009), fourth_free_address);
-
-        let (address, size) = lib.get_pub_allocation("my_thing");
-        assert_eq!(fourth_free_address, address);
-        assert_eq!(10_000, size);
     }
 }

tasm-lib/src/list/swap_unchecked.rs

@@ -174,7 +174,7 @@ impl Algorithm for SwapUnchecked {
     ) -> AlgorithmInitialState {
         let mut rng: StdRng = SeedableRng::from_seed(seed);
         let list_pointer = BFieldElement::new(rng.gen());
-        let list_length = rng.gen_range(0..200);
+        let list_length = rng.gen_range(1..200);
         let a = rng.gen_range(0..list_length);
         let b = rng.gen_range(0..list_length);
         self.initial_state(list_pointer, list_length, a, b)

tasm-lib/src/neptune.rs

@@ -1 +1,4 @@
 pub mod mutator_set;
+
+#[cfg(test)]
+pub(crate) mod neptune_like_types_for_tests;