refactor(Barretenberg): Static analysis of Protogalaxy Recursive Verifier (#16632)

This time Protogalaxy Recursive Verifier was tested.

1. The tool was modified in order to be able to process Ultra and Mega
Circuit Builders depending on the type of the circuit
2. The algorithm for finding connected components was fixed
3. Small refactoring: unused functions were removed and new added new
connected components algorithm, changed find_block_index to remove void*
:)
4. Tests for Protogalaxy Recursive Verifier were added for folding and
decider circuits. More false cases, 0 vulnerabilities.

Author: DanielKotov

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -13,7 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="d06b78a2"
+pinned_short_hash="98409939"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/boomerang_value_detection/CMakeLists.txt

@@ -1 +1,3 @@
-barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s stdlib_blake3s stdlib_poseidon2 stdlib_goblin_verifier)
+barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker 
+                    stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s 
+                    stdlib_blake3s stdlib_poseidon2 stdlib_honk_verifier stdlib_protogalaxy_verifier)

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph.cpp

@@ -1,4 +1,5 @@
 #pragma once
+#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 #include <list>
 #include <set>
@@ -9,8 +10,6 @@
 #include <vector>
 
 namespace cdg {
-
-using UltraBlock = bb::UltraTraceBlock;
 /**
  * We've added a new feature to the static analyzer that tracks which gates contain each variable.
  * This is helpful for removing false-positive variables from the analyzer by using gate selectors
@@ -49,6 +48,19 @@ struct KeyEquals {
     }
 };
 
+struct ConnectedComponent {
+    std::vector<uint32_t> variable_indices;
+    bool is_range_list_cc;
+    bool is_finalize_cc;
+    ConnectedComponent() = default;
+    ConnectedComponent(const std::vector<uint32_t>& vector)
+        : variable_indices(vector)
+        , is_range_list_cc(false)
+        , is_finalize_cc(false) {};
+    size_t size() const { return variable_indices.size(); }
+    const std::vector<uint32_t>& vars() const { return variable_indices; }
+};
+
 /*
  * This class describes an arithmetic circuit as an undirected graph, where vertices are variables from the circuit.
  * Edges describe connections between variables through gates. We want to find variables that weren't properly
@@ -57,133 +69,91 @@ struct KeyEquals {
  * variable wasn't constrained properly. If the number of connected components > 1, it means that there were some missed
  * connections between variables.
  */
-template <typename FF> class StaticAnalyzer_ {
+template <typename FF, typename CircuitBuilder> class StaticAnalyzer_ {
   public:
     StaticAnalyzer_() = default;
     StaticAnalyzer_(const StaticAnalyzer_& other) = delete;
     StaticAnalyzer_(StaticAnalyzer_&& other) = delete;
     StaticAnalyzer_& operator=(const StaticAnalyzer_& other) = delete;
     StaticAnalyzer_&& operator=(StaticAnalyzer_&& other) = delete;
-    StaticAnalyzer_(bb::UltraCircuitBuilder& ultra_circuit_constructor, bool connect_variables = true);
+    StaticAnalyzer_(CircuitBuilder& circuit_builder, bool connect_variables = true);
 
     /**
      * @brief Convert a vector of variable indices to their real indices
-     * @param ultra_circuit_constructor The UltraCircuitBuilder instance
      * @param variable_indices The vector of variable indices to convert
      * @return std::vector<uint32_t> A vector of real variable indices
      */
-    std::vector<uint32_t> to_real(bb::UltraCircuitBuilder& ultra_circuit_constructor,
-                                  std::vector<uint32_t>& variable_indices)
+    std::vector<uint32_t> to_real(std::vector<uint32_t>& variable_indices)
     {
         std::vector<uint32_t> real_variable_indices;
         real_variable_indices.reserve(variable_indices.size());
         for (auto& variable_index : variable_indices) {
-            real_variable_indices.push_back(to_real(ultra_circuit_constructor, variable_index));
+            real_variable_indices.push_back(to_real(variable_index));
         }
         return real_variable_indices;
     };
-
-    uint32_t to_real(bb::UltraCircuitBuilder& ultra_circuit_constructor, const uint32_t& variable_index)
+    uint32_t to_real(const uint32_t& variable_index) const
     {
-        return ultra_circuit_constructor.real_variable_index[variable_index];
-    };
-    size_t find_block_index(bb::UltraCircuitBuilder& ultra_builder, const UltraBlock& block);
+        return circuit_builder.real_variable_index[variable_index];
+    }
+    size_t find_block_index(const auto& block);
     void process_gate_variables(std::vector<uint32_t>& gate_variables, size_t gate_index, size_t blk_idx);
-    std::unordered_map<uint32_t, size_t> get_variables_gate_counts() { return this->variables_gate_counts; };
-
-    std::vector<std::vector<uint32_t>> get_arithmetic_gate_connected_component(
-        bb::UltraCircuitBuilder& ultra_circuit_builder, size_t index, size_t block_idx, UltraBlock& blk);
-    std::vector<uint32_t> get_elliptic_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                size_t index,
-                                                                size_t block_idx,
-                                                                UltraBlock& blk);
-    std::vector<uint32_t> get_plookup_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                               size_t index,
-                                                               size_t block_idx,
-                                                               UltraBlock& blk);
-    std::vector<uint32_t> get_sort_constraint_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                  size_t index,
-                                                                  size_t block_idx,
-                                                                  UltraBlock& blk);
-    std::vector<uint32_t> get_poseido2s_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                 size_t index,
-                                                                 size_t block_idx,
-                                                                 UltraBlock& blk);
-    std::vector<uint32_t> get_memory_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                              size_t index,
-                                                              size_t block_idx,
-                                                              UltraBlock& blk);
-    std::vector<uint32_t> get_non_native_field_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                        size_t index,
-                                                                        size_t block_idx,
-                                                                        UltraBlock& blk);
-    std::vector<uint32_t> get_rom_table_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                            const bb::RomTranscript& rom_array);
-    std::vector<uint32_t> get_ram_table_connected_component(bb::UltraCircuitBuilder& ultra_builder,
-                                                            const bb::RamTranscript& ram_array);
+    std::unordered_map<uint32_t, size_t> get_variables_gate_counts() const { return this->variables_gate_counts; };
+
+    void process_execution_trace();
+
+    std::vector<std::vector<uint32_t>> get_arithmetic_gate_connected_component(size_t index,
+                                                                               size_t block_idx,
+                                                                               auto& blk);
+    std::vector<uint32_t> get_elliptic_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_plookup_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_sort_constraint_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_poseido2s_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_non_native_field_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_memory_gate_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_rom_table_connected_component(const bb::RomTranscript& rom_array);
+    std::vector<uint32_t> get_ram_table_connected_component(const bb::RamTranscript& ram_array);
+    // functions for MegaCircuitBuilder
+    std::vector<uint32_t> get_databus_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_eccop_connected_component(size_t index, size_t block_idx, auto& blk);
+    std::vector<uint32_t> get_eccop_part_connected_component(size_t index, size_t block_idx, auto& blk);
 
     void add_new_edge(const uint32_t& first_variable_index, const uint32_t& second_variable_index);
-    std::vector<uint32_t> get_variable_adjacency_list(const uint32_t& variable_index)
-    {
-        return variable_adjacency_lists[variable_index];
-    };
-
     void depth_first_search(const uint32_t& variable_index,
                             std::unordered_set<uint32_t>& is_used,
                             std::vector<uint32_t>& connected_component);
-    std::vector<std::vector<uint32_t>> find_connected_components();
-
-    std::vector<uint32_t> find_variables_with_degree_one();
-    std::unordered_set<uint32_t> get_variables_in_one_gate();
-
-    bool find_arithmetic_gate_for_variable(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                           const uint32_t& variable_idx);
-    bool find_elliptic_gate_for_variable(bb::UltraCircuitBuilder& ultra_circuit_builder, const uint32_t& variable_idx);
-    bool find_lookup_gate_for_variable(bb::UltraCircuitBuilder& ultra_circuit_builder, const uint32_t& variable_idx);
-
-    size_t get_distance_between_variables(const uint32_t& first_variable_index, const uint32_t& second_variable_index);
+    void mark_range_list_connected_components();
+    void mark_finalize_connected_components();
+    std::vector<ConnectedComponent> find_connected_components(bool return_all_connected_components = false);
     bool check_vertex_in_connected_component(const std::vector<uint32_t>& connected_component,
                                              const uint32_t& var_index);
-
-    void connect_all_variables_in_vector(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                         const std::vector<uint32_t>& variables_vector);
-    bool check_is_not_constant_variable(bb::UltraCircuitBuilder& ultra_circuit_builder, const uint32_t& variable_index);
+    void connect_all_variables_in_vector(const std::vector<uint32_t>& variables_vector);
+    bool check_is_not_constant_variable(const uint32_t& variable_index);
 
     std::pair<std::vector<uint32_t>, size_t> get_connected_component_with_index(
         const std::vector<std::vector<uint32_t>>& connected_components, size_t index);
 
-    std::unordered_set<uint32_t> get_variables_in_one_gate_without_range_constraints(
-        bb::UltraCircuitBuilder& ultra_circuit_builder);
-
-    size_t process_current_decompose_chain(bb::UltraCircuitBuilder& ultra_circuit_constructor,
-                                           std::unordered_set<uint32_t>& variables_in_one_gate,
-                                           size_t index);
-    void process_current_plookup_gate(bb::UltraCircuitBuilder& ultra_circuit_builder, size_t gate_index);
-    void remove_unnecessary_decompose_variables(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                std::unordered_set<uint32_t>& variables_in_on_gate,
-                                                const std::unordered_set<uint32_t>& decompose_variables);
-    void remove_unnecessary_plookup_variables(bb::UltraCircuitBuilder& ultra_circuit_builder);
-    void remove_unnecessary_range_constrains_variables(bb::UltraCircuitBuilder& ultra_builder);
-    std::unordered_set<uint32_t> show_variables_in_one_gate(bb::UltraCircuitBuilder& ultra_circuit_builder);
-
-    void remove_unnecessary_aes_plookup_variables(std::unordered_set<uint32_t>& variables_in_one_gate,
-                                                  bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                  bb::plookup::BasicTableId& table_id,
-                                                  size_t gate_index);
-    void remove_unnecessary_sha256_plookup_variables(std::unordered_set<uint32_t>& variables_in_one_gate,
-                                                     bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                     bb::plookup::BasicTableId& table_id,
-                                                     size_t gate_index);
-    void remove_record_witness_variables(bb::UltraCircuitBuilder& ultra_builder);
-
-    void print_graph();
-    void print_connected_components();
+    size_t process_current_decompose_chain(size_t index);
+    void process_current_plookup_gate(size_t gate_index);
+    void remove_unnecessary_decompose_variables(const std::unordered_set<uint32_t>& decompose_variables);
+    void remove_unnecessary_plookup_variables();
+    void remove_unnecessary_range_constrains_variables();
+    std::unordered_set<uint32_t> get_variables_in_one_gate();
+
+    void remove_unnecessary_aes_plookup_variables(bb::plookup::BasicTableId& table_id, size_t gate_index);
+    void remove_unnecessary_sha256_plookup_variables(bb::plookup::BasicTableId& table_id, size_t gate_index);
+    void remove_record_witness_variables();
+
+    void print_connected_components_info();
     void print_variables_gate_counts();
-    void print_variables_edge_counts();
-    void print_variable_in_one_gate(bb::UltraCircuitBuilder& ultra_builder, const uint32_t real_idx);
+    void print_variable_in_one_gate(const uint32_t real_idx);
     ~StaticAnalyzer_() = default;
 
   private:
+    // Store reference to the circuit builder
+    CircuitBuilder& circuit_builder;
+    bool connect_variables;
+
     std::unordered_map<uint32_t, std::vector<uint32_t>>
         variable_adjacency_lists; // we use this data structure to contain information about variables and their
                                   // connections between each other
@@ -193,11 +163,17 @@ template <typename FF> class StaticAnalyzer_ {
         variables_degree; // we use this data structure to count, how many every variable have edges
     std::unordered_map<KeyPair, std::vector<size_t>, KeyHasher, KeyEquals>
         variable_gates; // we use this data structure to store gates and TraceBlocks for every variables, where static
-                        // analyzer found them in the circuit.
+                        // analyzer finds them in the circuit.
     std::unordered_set<uint32_t> variables_in_one_gate;
     std::unordered_set<uint32_t> fixed_variables;
+    std::vector<ConnectedComponent> connected_components;
+    std::vector<ConnectedComponent>
+        main_connected_components; // connected components without finalize blocks and range lists
 };
 
-using StaticAnalyzer = StaticAnalyzer_<bb::fr>;
+// Type aliases for convenience
+using UltraStaticAnalyzer = StaticAnalyzer_<bb::fr, bb::UltraCircuitBuilder>;
+using MegaStaticAnalyzer = StaticAnalyzer_<bb::fr, bb::MegaCircuitBuilder>;
+using StaticAnalyzer = UltraStaticAnalyzer; // Default to Ultra for backward compatibility
 
 } // namespace cdg