577 new fast tbl

.github/workflows/build_linux.yml

@@ -327,11 +327,11 @@ jobs:
       always() && !cancelled() &&
       (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
       needs.build-and-test-linux.result == 'success'
-    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@ef8cd9152b4bec871e7efdc1d6b606e445bad274
+    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@f450e9219b0b77fa32ce27c5e9a9d9144cc577dd
     with:
       artifact-name: ${{ needs.build-and-test-linux.outputs.examples-artifact-name }}
       # Update next line if you need new version of proof producer
-      proof-producer-ref: ef8cd9152b4bec871e7efdc1d6b606e445bad274
+      proof-producer-ref: f450e9219b0b77fa32ce27c5e9a9d9144cc577dd
       refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
       targets: ${{ needs.build-and-test-linux.outputs.prover-targets }}
 

bin/recursive_gen/src/main.cpp

@@ -53,6 +53,8 @@
 #include <nil/blueprint/transpiler/public_input.hpp>
 #include <nil/blueprint/transpiler/recursive_verifier_generator.hpp>
 
+#include "../../table_packing.hpp"
+
 template<typename StreamType>
 std::optional<StreamType> open_file(const std::string& path, std::ios_base::openmode mode) {
     StreamType file(path, mode);
@@ -106,6 +108,31 @@ std::optional<MarshallingType> decode_marshalling_from_file(
     return marshalled_data;
 }
 
+template<typename MarshallingType>
+std::optional<MarshallingType> decode_marshalling_from_different_column_types_files(
+    const std::string& assignment_table_file_name
+) {
+
+        std::vector<std::uint8_t> v = {};
+
+        ASSERT(append_binary_file_content_to_vector(v, "header_", assignment_table_file_name));
+        ASSERT(append_binary_file_content_to_vector(v, "witness_", assignment_table_file_name));
+        ASSERT(append_binary_file_content_to_vector(v, "pub_inp_", assignment_table_file_name));
+        ASSERT(append_binary_file_content_to_vector(v, "constants_", assignment_table_file_name));
+        ASSERT(append_binary_file_content_to_vector(v, "selectors_", assignment_table_file_name));
+
+        MarshallingType marshalled_data;
+        auto read_iter = v.begin();
+        auto status = marshalled_data.read(read_iter, v.size());
+
+        if (status != nil::marshalling::status_type::success) {
+            std::cerr << "Marshalled structure decoding failed" << std::endl;
+            return std::nullopt;
+        }
+        return marshalled_data;
+}
+
+
 template<typename BlueprintFieldType>
 struct ParametersPolicy {
     constexpr static const std::size_t WitnessColumns = WITNESS_COLUMNS;
@@ -414,7 +441,7 @@ int curve_dependent_main(
     public_input_sizes = (*constraint_system).public_input_sizes();
 
     if( vm.count("assignment-table") ){
-        auto marshalled_value = decode_marshalling_from_file<assignment_table_marshalling_type>(assignment_table_file_name);
+        auto marshalled_value = decode_marshalling_from_different_column_types_files<assignment_table_marshalling_type>(assignment_table_file_name);
         if (!marshalled_value) {
             return false;
         }

bin/table_packing.hpp

@@ -0,0 +1,73 @@
+std::string add_filename_prefix(
+    const std::string& prefix,
+    const std::string& file_name
+) {
+    std::filesystem::path path(file_name);
+    std::filesystem::path parent_path = path.parent_path();
+    std::filesystem::path filename = path.filename();
+
+    std::string new_filename = prefix + filename.string();
+    std::filesystem::path new_path = parent_path / new_filename;
+
+    return new_path.string();
+}
+
+
+bool append_binary_file_content_to_vector (
+    std::vector<std::uint8_t>& result_vector,
+    const std::string& prefix,
+    const std::string& assignment_table_file_name
+
+) {
+    std::ifstream icolumn;
+    icolumn.open(add_filename_prefix(prefix, assignment_table_file_name), std::ios_base::binary | std::ios_base::in);
+    if (!icolumn) {
+        std::cout << "Cannot open " << add_filename_prefix(prefix, assignment_table_file_name) << std::endl;
+        return false;
+    }
+    icolumn.seekg(0, std::ios_base::end);
+    const auto input_size = icolumn.tellg();
+    std::size_t old_size = result_vector.size();
+    result_vector.resize(old_size + input_size);
+    icolumn.seekg(0, std::ios_base::beg);
+    icolumn.read(reinterpret_cast<char*>(result_vector.data() + old_size), input_size);
+    icolumn.close();
+    return true;
+}
+
+template<typename marshalling_type>
+void unmarshall_from_vector (
+    std::vector<std::uint8_t>& v,
+    marshalling_type& marhsalling_data
+) {
+    auto vect_iterator = v.begin();
+    auto marshalling_status = marhsalling_data.read(vect_iterator, v.size());
+    ASSERT(marshalling_status == nil::marshalling::status_type::success);
+}
+
+template<typename marshalling_type>
+marshalling_type extract_from_binary_file(
+    const std::string& prefix,
+    const std::string& file_name
+) {
+    std::vector<std::uint8_t> file_vector = {};
+    ASSERT(append_binary_file_content_to_vector(file_vector, prefix, file_name));
+    marshalling_type marshalling_file;
+    unmarshall_from_vector<marshalling_type> (file_vector, marshalling_file);
+    return marshalling_file;
+}
+
+template<typename marshalling_type>
+marshalling_type extract_table_from_binary_file(
+    const std::string& file_name
+) {
+    std::vector<std::uint8_t> file_vector = {};
+    ASSERT(append_binary_file_content_to_vector(file_vector, "header_", file_name));
+    ASSERT(append_binary_file_content_to_vector(file_vector, "witness_", file_name));
+    ASSERT(append_binary_file_content_to_vector(file_vector, "pub_inp_", file_name));
+    ASSERT(append_binary_file_content_to_vector(file_vector, "constants_", file_name));
+    ASSERT(append_binary_file_content_to_vector(file_vector, "selectors_", file_name));
+    marshalling_type marshalling_data;
+    unmarshall_from_vector<marshalling_type> (file_vector, marshalling_data);
+    return marshalling_data;
+}

bin/transpiler/src/main.cpp

@@ -45,6 +45,8 @@
 #include <nil/blueprint/transpiler/evm_verifier_gen.hpp>
 #include <nil/blueprint/transpiler/public_input.hpp>
 
+#include "../../table_packing.hpp"
+
 template<typename BlueprintFieldType, typename ConstraintSystemType, typename ColumnsRotationsType>
 void print_sol_files(
     zk::snark::plonk_table_description<BlueprintFieldType> desc,
@@ -242,6 +244,7 @@ int main(int argc, char *argv[]) {
 
 }
 
+
 template<typename BlueprintFieldType, bool is_multi_prover>
 int curve_dependent_main(
     boost::program_options::options_description options_desc,
@@ -361,26 +364,11 @@ int curve_dependent_main(
 
     AssignmentTableType assignment_table;
     {
-        std::ifstream iassignment;
-        iassignment.open(assignment_table_file_name, std::ios_base::binary | std::ios_base::in);
-        if (!iassignment) {
-            std::cout << "Cannot open " << assignment_table_file_name << std::endl;
-            return 1;
-        }
-        std::vector<std::uint8_t> v;
-        iassignment.seekg(0, std::ios_base::end);
-        const auto fsize = iassignment.tellg();
-        v.resize(fsize);
-        iassignment.seekg(0, std::ios_base::beg);
-        iassignment.read(reinterpret_cast<char*>(v.data()), fsize);
-        if (!iassignment) {
-            std::cout << "Cannot parse input file " << assignment_table_file_name << std::endl;
-            return 1;
-        }
-        iassignment.close();
-        table_value_marshalling_type marshalled_table_data;
-        auto read_iter = v.begin();
-        auto status = marshalled_table_data.read(read_iter, v.size());
+
+        table_value_marshalling_type marshalled_table_data =
+            extract_table_from_binary_file<table_value_marshalling_type>
+                (assignment_table_file_name);
+
         std::tie(desc, assignment_table) =
             nil::crypto3::marshalling::types::make_assignment_table<Endianness, AssignmentTableType>(
                 marshalled_table_data

examples/CMakeLists.txt

@@ -33,7 +33,11 @@ function(assign_ir)
     add_custom_target(${target}_generate_crct
             COMMAND $<TARGET_FILE:assigner>
             -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            ${minus_p} ${private_input_string}
             -c circuit_${target}.crct
+            -j table_pieces_${target}.json
+            -t assignment_${target}.tbl
             -e ${curve_type}
             --generate-type circuit
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -48,6 +52,7 @@ function(assign_ir)
             -i ${INPUTS_DIR}/${input}
             ${minus_p} ${private_input_string}
             -c circuit_${target}.crct
+            -j table_pieces_${target}.json
             -t assignment_${target}.tbl -e ${curve_type} --check
             --generate-type circuit-assignment
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -62,6 +67,7 @@ function(assign_ir)
             -i ${INPUTS_DIR}/${input}
             ${minus_p} ${private_input_string}
             -c circuit_${target}.crct
+            -j table_pieces_${target}.json
             -t assignment_${target}.tbl -e ${curve_type}
             --generate-type circuit-assignment
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -83,7 +89,35 @@ function(assign_ir)
         COMMAND_EXPAND_LISTS
         VERBATIM)
 
-    add_custom_target(${target}_generate_tbl_no_check
+    add_custom_target(${target}_generate_tbl_fast
+        COMMAND $<TARGET_FILE:assigner>
+            -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            ${minus_p} ${private_input_string}
+            -j table_pieces_${target}.json
+            -t assignment_${target}.tbl -e ${curve_type}
+            --generate-type assignment-fast
+            ${max_num_provers_flag} ${max_num_provers_amount}
+            ${arithmetization_flag} ${arithmetization_amount}
+        DEPENDS ${target} ${INPUTS_DIR}/${input} $<TARGET_FILE:assigner>
+        COMMAND_EXPAND_LISTS
+        VERBATIM)
+
+    add_custom_target(${target}_generate_tbl_fast_depends_on_crct
+        COMMAND $<TARGET_FILE:assigner>
+            -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            ${minus_p} ${private_input_string}
+            -j table_pieces_${target}.json
+            -t assignment_${target}.tbl -e ${curve_type}
+            --generate-type assignment-fast
+            ${max_num_provers_flag} ${max_num_provers_amount}
+            ${arithmetization_flag} ${arithmetization_amount}
+        DEPENDS ${target} ${INPUTS_DIR}/${input} ${target}_generate_crct $<TARGET_FILE:assigner>
+        COMMAND_EXPAND_LISTS
+        VERBATIM)
+
+        add_custom_target(${target}_generate_tbl_no_check
         COMMAND $<TARGET_FILE:assigner>
             -b ${binary_name}
             -i ${INPUTS_DIR}/${input}
@@ -99,6 +133,9 @@ function(assign_ir)
     add_custom_target(${target}_estimate_size
         COMMAND $<TARGET_FILE:assigner>
             -b ${binary_name}
+            -i ${INPUTS_DIR}/${input}
+            -j table_pieces_${target}.json
+            ${minus_p} ${private_input_string}
             -e ${curve_type}
             --generate-type size_estimation
             ${max_num_provers_flag} ${max_num_provers_amount}
@@ -120,15 +157,15 @@ function(assign_ir)
         VERBATIM)
 endfunction()
 
-function(gen_proof target curve_type provers_amount)
+function(gen_proof target curve_type provers_amount tbl_gen_speed)
     if(provers_amount EQUAL 0)
-        gen_single_proof(${target} ${curve_type} 0)
+        gen_single_proof(${target} ${curve_type} 0 ${tbl_gen_speed})
     else()
         add_custom_target(${target}_prove)
 
         foreach(prover_num RANGE 1 ${provers_amount})
             math(EXPR prover_num_minus_1 "${prover_num} - 1")
-            gen_single_proof(${target} ${curve_type} ${prover_num})
+            gen_single_proof(${target} ${curve_type} ${prover_num} ${tbl_gen_speed})
             add_dependencies(${target}_prove ${target}_prove${prover_num_minus_1})
         endforeach()
 
@@ -143,22 +180,27 @@ function(gen_proof target curve_type provers_amount)
 endfunction()
 
 
-function(gen_single_proof target curve_type provers_amount)
+function(gen_single_proof target curve_type provers_amount tbl_gen_speed)
     if(NOT provers_amount EQUAL 0)
         set(multi_prover_flag --multi-prover)
         math(EXPR prover_num "${provers_amount} - 1")
     else()
         set(prover_num "")
     endif()
 
+    set (depends_tbl ${target}_generate_both)
+    if (tbl_gen_speed STREQUAL "fast")
+        set (depends_tbl ${target}_generate_tbl_fast_depends_on_crct)
+    endif()
+
     add_custom_target(${target}_prove${prover_num}
         COMMAND $<TARGET_FILE:transpiler> -m gen-test-proof
             -c circuit_${target}.crct${prover_num}
             -t assignment_${target}.tbl${prover_num}
             -o transpiler_output_${target}${prover_num}
             -e ${curve_type}
             ${multi_prover_flag}
-            DEPENDS ${target}_generate_crct ${target}_generate_tbl $<TARGET_FILE:transpiler>
+            DEPENDS ${depends_tbl} $<TARGET_FILE:transpiler>
         COMMAND_EXPAND_LISTS
         VERBATIM)
 endfunction()