1 big commit for graph description of protogalaxy_recursive_verifier

Author: DanielKotov

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -13,7 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="d06b78a2"
+pinned_short_hash="98409939"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/boomerang_value_detection/CMakeLists.txt

@@ -1 +1,3 @@
-barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s stdlib_blake3s stdlib_poseidon2 stdlib_goblin_verifier)
+barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker 
+                    stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s 
+                    stdlib_blake3s stdlib_poseidon2 stdlib_honk_verifier stdlib_protogalaxy_verifier)

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph.cpp

@@ -1,4 +1,5 @@
 #pragma once
+#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 #include <list>
 #include <set>
@@ -9,8 +10,6 @@
 #include <vector>
 
 namespace cdg {
-
-using UltraBlock = bb::UltraTraceBlock;
 /**
  * We've added a new feature to the static analyzer that tracks which gates contain each variable.
  * This is helpful for removing false-positive variables from the analyzer by using gate selectors
@@ -49,6 +48,19 @@ struct KeyEquals {
     }
 };
 
+struct ConnectedComponent {
+    std::vector<uint32_t> variable_indices;
+    bool is_range_list_cc;
+    bool is_finalize_cc;
+    ConnectedComponent() = default;
+    ConnectedComponent(const std::vector<uint32_t>& vector)
+        : variable_indices(vector)
+        , is_range_list_cc(false)
+        , is_finalize_cc(false) {};
+    size_t size() const { return variable_indices.size(); }
+    const std::vector<uint32_t>& vars() const { return variable_indices; }
+};
+
 /*
  * This class describes an arithmetic circuit as an undirected graph, where vertices are variables from the circuit.
  * Edges describe connections between variables through gates. We want to find variables that weren't properly
@@ -57,133 +69,91 @@ struct KeyEquals {
  * variable wasn't constrained properly. If the number of connected components > 1, it means that there were some missed
  * connections between variables.
  */
-template <typename FF> class StaticAnalyzer_ {
+template <typename FF, typename CircuitBuilder> class StaticAnalyzer_ {
   public:
     StaticAnalyzer_() = default;
     StaticAnalyzer_(const StaticAnalyzer_& other) = delete;
     StaticAnalyzer_(StaticAnalyzer_&& other) = delete;
     StaticAnalyzer_& operator=(const StaticAnalyzer_& other) = delete;
     StaticAnalyzer_&& operator=(StaticAnalyzer_&& other) = delete;
-    StaticAnalyzer_(bb::UltraCircuitBuilder& ultra_circuit_constructor, bool connect_variables = true);
+    StaticAnalyzer_(CircuitBuilder& circuit_builder, bool connect_variables = true);
 
     /**
      * @brief Convert a vector of variable indices to their real indices
-     * @param ultra_circuit_constructor The UltraCircuitBuilder instance
      * @param variable_indices The vector of variable indices to convert
      * @return std::vector<uint32_t> A vector of real variable indices
      */
-    std::vector<uint32_t> to_real(bb::UltraCircuitBuilder& ultra_circuit_constructor,
-                                  std::vector<uint32_t>& variable_indices)
+    std::vector<uint32_t> to_real(std::vector<uint32_t>& variable_indices)
     {
         std::vector<uint32_t> real_variable_indices;
         real_variable_indices.reserve(variable_indices.size());
         for (auto& variable_index : variable_indices) {
-            real_variable_indices.push_back(to_real(ultra_circuit_constructor, variable_index));
+            real_variable_indices.push_back(to_real(variable_index));
         }
         return real_variable_indices;
     };
-
-    uint32_t to_real(bb::UltraCircuitBuilder& ultra_circuit_constructor, const uint32_t& variable_index)
+    uint32_t to_real(const uint32_t& variable_index) const
     {
-        return ultra_circuit_constructor.real_variable_index[variable_index];
-    };
-    size_t find_block_index(bb::UltraCircuitBuilder& ultra_builder, const UltraBlock& block);
+        return circuit_builder.real_variable_index[variable_index];
+    }
+    size_t find_block_index(const auto& block);
     void process_gate_variables(std::vector<uint32_t>& gate_variables, size_t gate_index, size_t blk_idx);
-    std::unordered_map<uint32_t, size_t> get_variables_gate_counts() { return this->variables_gate_counts; };
-
-    std::vector<std::vector<uint32_t>> get_arithmetic_gate_connected_component(
-        bb::UltraCircuitBuilder& ultra_circuit_builder, size_t index, size_t block_idx, UltraBlock& blk);
-    std::vector<uint32_t> get_elliptic_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                size_t index,
-                                                                size_t block_idx,
-                                                                UltraBlock& blk);
-    std::vector<uint32_t> get_plookup_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                               size_t index,
-                                                               size_t block_idx,
-                                                               UltraBlock& blk);
-    std::vector<uint32_t> get_sort_constraint_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                  size_t index,
-                                                                  size_t block_idx,
-                                                                  UltraBlock& blk);
-    std::vector<uint32_t> get_poseido2s_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                 size_t index,
-                                                                 size_t block_idx,
-                                                                 UltraBlock& blk);
-    std::vector<uint32_t> get_memory_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                              size_t index,
-                                                              size_t block_idx,
-                                                              UltraBlock& blk);
-    std::vector<uint32_t> get_non_native_field_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                        size_t index,
-                                                                        size_t block_idx,
-                                                                        UltraBlock& blk);
-    std::vector<uint32_t> get_rom_table_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                            const bb::RomTranscript& rom_array);
-    std::vector<uint32_t> get_ram_table_connected_component(bb::UltraCircuitBuilder& ultra_builder,
-                                                            const bb::RamTranscript& ram_array);
+    std::unordered_map<uint32_t, size_t> get_variables_gate_counts() const { return this->variables_gate_counts; };
+
+    void process_execution_trace();
+
+    std::vector<std::vector<uint32_t>> get_arithmetic_gate_connected_component(size_t index,
+                                                                               size_t block_idx,
+                                                                               auto& blk);
+    std::vector<uint32_t> get_elliptic_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_plookup_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_sort_constraint_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_poseido2s_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_non_native_field_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_memory_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_rom_table_connected_component(const bb::RomTranscript& rom_array);
+    std::vector<uint32_t> get_ram_table_connected_component(const bb::RamTranscript& ram_array);
+    // functions for MegaCircuitBuilder
+    std::vector<uint32_t> get_databus_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_eccop_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_eccop_part_connected_component(size_t index, size_t block_idx, auto& blk);
 
     void add_new_edge(const uint32_t& first_variable_index, const uint32_t& second_variable_index);
-    std::vector<uint32_t> get_variable_adjacency_list(const uint32_t& variable_index)
-    {
-        return variable_adjacency_lists[variable_index];
-    };
-
     void depth_first_search(const uint32_t& variable_index,
                             std::unordered_set<uint32_t>& is_used,
                             std::vector<uint32_t>& connected_component);
-    std::vector<std::vector<uint32_t>> find_connected_components();
-
-    std::vector<uint32_t> find_variables_with_degree_one();
-    std::unordered_set<uint32_t> get_variables_in_one_gate();
-
-    bool find_arithmetic_gate_for_variable(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                           const uint32_t& variable_idx);
-    bool find_elliptic_gate_for_variable(bb::UltraCircuitBuilder& ultra_circuit_builder, const uint32_t& variable_idx);
-    bool find_lookup_gate_for_variable(bb::UltraCircuitBuilder& ultra_circuit_builder, const uint32_t& variable_idx);
-
-    size_t get_distance_between_variables(const uint32_t& first_variable_index, const uint32_t& second_variable_index);
+    void mark_range_list_connected_components();
+    void mark_finalize_connected_components();
+    std::vector<ConnectedComponent> find_connected_components(bool return_all_connected_components = false);
     bool check_vertex_in_connected_component(const std::vector<uint32_t>& connected_component,
                                              const uint32_t& var_index);
-
-    void connect_all_variables_in_vector(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                         const std::vector<uint32_t>& variables_vector);
-    bool check_is_not_constant_variable(bb::UltraCircuitBuilder& ultra_circuit_builder, const uint32_t& variable_index);
+    void connect_all_variables_in_vector(const std::vector<uint32_t>& variables_vector);
+    bool check_is_not_constant_variable(const uint32_t& variable_index);
 
     std::pair<std::vector<uint32_t>, size_t> get_connected_component_with_index(
         const std::vector<std::vector<uint32_t>>& connected_components, size_t index);
 
-    std::unordered_set<uint32_t> get_variables_in_one_gate_without_range_constraints(
-        bb::UltraCircuitBuilder& ultra_circuit_builder);
-
-    size_t process_current_decompose_chain(bb::UltraCircuitBuilder& ultra_circuit_constructor,
-                                           std::unordered_set<uint32_t>& variables_in_one_gate,
-                                           size_t index);
-    void process_current_plookup_gate(bb::UltraCircuitBuilder& ultra_circuit_builder, size_t gate_index);
-    void remove_unnecessary_decompose_variables(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                std::unordered_set<uint32_t>& variables_in_on_gate,
-                                                const std::unordered_set<uint32_t>& decompose_variables);
-    void remove_unnecessary_plookup_variables(bb::UltraCircuitBuilder& ultra_circuit_builder);
-    void remove_unnecessary_range_constrains_variables(bb::UltraCircuitBuilder& ultra_builder);
-    std::unordered_set<uint32_t> show_variables_in_one_gate(bb::UltraCircuitBuilder& ultra_circuit_builder);
-
-    void remove_unnecessary_aes_plookup_variables(std::unordered_set<uint32_t>& variables_in_one_gate,
-                                                  bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                  bb::plookup::BasicTableId& table_id,
-                                                  size_t gate_index);
-    void remove_unnecessary_sha256_plookup_variables(std::unordered_set<uint32_t>& variables_in_one_gate,
-                                                     bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                     bb::plookup::BasicTableId& table_id,
-                                                     size_t gate_index);
-    void remove_record_witness_variables(bb::UltraCircuitBuilder& ultra_builder);
-
-    void print_graph();
-    void print_connected_components();
+    size_t process_current_decompose_chain(size_t index);
+    void process_current_plookup_gate(size_t gate_index);
+    void remove_unnecessary_decompose_variables(const std::unordered_set<uint32_t>& decompose_variables);
+    void remove_unnecessary_plookup_variables();
+    void remove_unnecessary_range_constrains_variables();
+    std::unordered_set<uint32_t> get_variables_in_one_gate();
+
+    void remove_unnecessary_aes_plookup_variables(bb::plookup::BasicTableId& table_id, size_t gate_index);
+    void remove_unnecessary_sha256_plookup_variables(bb::plookup::BasicTableId& table_id, size_t gate_index);
+    void remove_record_witness_variables();
+
+    void print_connected_components_info();
     void print_variables_gate_counts();
-    void print_variables_edge_counts();
-    void print_variable_in_one_gate(bb::UltraCircuitBuilder& ultra_builder, const uint32_t real_idx);
+    void print_variable_in_one_gate(const uint32_t real_idx);
     ~StaticAnalyzer_() = default;
 
   private:
+    // Store reference to the circuit builder
+    CircuitBuilder& circuit_builder;
+    bool connect_variables;
+
     std::unordered_map<uint32_t, std::vector<uint32_t>>
         variable_adjacency_lists; // we use this data structure to contain information about variables and their
                                   // connections between each other
@@ -193,11 +163,17 @@ template <typename FF> class StaticAnalyzer_ {
         variables_degree; // we use this data structure to count, how many every variable have edges
     std::unordered_map<KeyPair, std::vector<size_t>, KeyHasher, KeyEquals>
         variable_gates; // we use this data structure to store gates and TraceBlocks for every variables, where static
-                        // analyzer found them in the circuit.
+                        // analyzer finds them in the circuit.
     std::unordered_set<uint32_t> variables_in_one_gate;
     std::unordered_set<uint32_t> fixed_variables;
+    std::vector<ConnectedComponent> connected_components;
+    std::vector<ConnectedComponent>
+        main_connected_components; // connected components without finalize blocks and range lists
 };
 
-using StaticAnalyzer = StaticAnalyzer_<bb::fr>;
+// Type aliases for convenience
+using UltraStaticAnalyzer = StaticAnalyzer_<bb::fr, bb::UltraCircuitBuilder>;
+using MegaStaticAnalyzer = StaticAnalyzer_<bb::fr, bb::MegaCircuitBuilder>;
+using StaticAnalyzer = UltraStaticAnalyzer; // Default to Ultra for backward compatibility
 
 } // namespace cdg