Merge branch 'main' into np/refactoring

Author: nischitpra

internal/tools/clickhouse/0002_clickhouse_create_logs_table.sql

@@ -37,16 +37,23 @@ CREATE TABLE IF NOT EXISTS logs (
             transaction_index,
             log_index
     ),
-    PROJECTION chain_topic0_projection
+    PROJECTION chain_address_block_number_full_projection
+    (
+        SELECT 
+            *
+        ORDER BY
+            chain_id,
+            address,
+            block_number
+    ),
+    PROJECTION chain_topic0_full_projection
     (
         SELECT
-            _part_offset
+            *
         ORDER BY 
             chain_id,
             topic_0,
             block_number,
-            transaction_index,
-            log_index,
             address
     ),
     PROJECTION address_topic0_state_projection

internal/tools/clickhouse/0006_clickhouse_create_token_transfers.sql

@@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS token_transfers
     `insert_timestamp` DateTime DEFAULT now(),
     `is_deleted` UInt8 DEFAULT 0,
 
+    INDEX idx_block_number block_number TYPE minmax GRANULARITY 1,
     INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
     INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 3,
     INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 3,

internal/tools/clickhouse/0008_clickhouse_create_token_balances.sql

@@ -20,6 +20,7 @@ CREATE TABLE IF NOT EXISTS token_balances
   `insert_timestamp` DateTime DEFAULT now(),
   `is_deleted` UInt8 DEFAULT 0,
 
+  INDEX idx_block_number block_number TYPE minmax GRANULARITY 1,
   INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
   INDEX idx_token_address token_address TYPE bloom_filter GRANULARITY 3,
   INDEX idx_owner_address owner_address TYPE bloom_filter GRANULARITY 3,
@@ -31,11 +32,11 @@ CREATE TABLE IF NOT EXISTS token_balances
       owner_address,
       token_address,
       token_id,
-      sumState(balance_delta * if(is_deleted = 0, 1, -1)) AS balance_state,
-      minState(block_number) AS min_block_number_state,
-      minState(block_timestamp) AS min_block_timestamp_state,
-      maxState(block_number) AS max_block_number_state,
-      maxState(block_timestamp) AS max_block_timestamp_state
+      sum(balance_delta * if(is_deleted = 0, 1, -1)) AS balance_state,
+      min(block_number) AS min_block_number_state,
+      min(block_timestamp) AS min_block_timestamp_state,
+      max(block_number) AS max_block_number_state,
+      max(block_timestamp) AS max_block_timestamp_state
     GROUP BY chain_id, owner_address, token_address, token_id
   ),
 
@@ -46,11 +47,11 @@ CREATE TABLE IF NOT EXISTS token_balances
       token_address,
       token_id,
       owner_address,
-      sumState(balance_delta * if(is_deleted = 0, 1, -1)) AS balance_state,
-      minState(block_number) AS min_block_number_state,
-      minState(block_timestamp) AS min_block_timestamp_state,
-      maxState(block_number) AS max_block_number_state,
-      maxState(block_timestamp) AS max_block_timestamp_state
+      sum(balance_delta * if(is_deleted = 0, 1, -1)) AS balance_state,
+      min(block_number) AS min_block_number_state,
+      min(block_timestamp) AS min_block_timestamp_state,
+      max(block_number) AS max_block_number_state,
+      max(block_timestamp) AS max_block_timestamp_state
     GROUP BY chain_id, token_address, token_id, owner_address
   ),
 
@@ -62,6 +63,6 @@ CREATE TABLE IF NOT EXISTS token_balances
   )
 )
 ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
-PARTITION BY chain_id
+PARTITION BY (chain_id, toStartOfQuarter(block_timestamp))
 ORDER BY (chain_id, owner_address, token_address, token_id, block_number, transaction_index, log_index, direction)
 SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild', allow_part_offset_column_in_projections=1;

internal/tools/clickhouse/0009_clickhouse_create_token_balances_mv.sql

@@ -1,8 +1,7 @@
 -- ERC20
-CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc20_mv
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc20_from_mv
 TO token_balances
 AS
--- FROM side (outgoing, negative delta)
 SELECT
   chain_id,
   token_type,
@@ -19,8 +18,11 @@ SELECT
   insert_timestamp,
   is_deleted
 FROM token_transfers 
-WHERE token_type = 'erc20'
-UNION ALL
+WHERE token_type = 'erc20';
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc20_to_mv
+TO token_balances
+AS
 -- TO side (incoming, positive delta)
 SELECT
   chain_id,
@@ -41,7 +43,7 @@ FROM token_transfers
 WHERE token_type = 'erc20';
 
 -- ERC721
-CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc721_mv
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc721_from_mv
 TO token_balances
 AS
 SELECT
@@ -60,8 +62,11 @@ SELECT
   insert_timestamp,
   is_deleted
 FROM token_transfers 
-WHERE token_type = 'erc721'
-UNION ALL
+WHERE token_type = 'erc721';
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc721_to_mv
+TO token_balances
+AS
 SELECT
   chain_id,
   token_type,
@@ -81,7 +86,7 @@ FROM token_transfers
 WHERE token_type = 'erc721';
 
 -- ERC1155
-CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc1155_mv
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc1155_from_mv
 TO token_balances
 AS
 SELECT
@@ -100,8 +105,11 @@ SELECT
   insert_timestamp,
   is_deleted
 FROM token_transfers 
-WHERE token_type = 'erc1155'
-UNION ALL
+WHERE token_type = 'erc1155';
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc1155_to_mv
+TO token_balances
+AS
 SELECT
   chain_id,
   token_type,
@@ -121,7 +129,7 @@ FROM token_transfers
 WHERE token_type = 'erc1155';
 
 -- ERC6909
-CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc6909_mv
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc6909_from_mv
 TO token_balances
 AS
 SELECT
@@ -140,8 +148,11 @@ SELECT
   insert_timestamp,
   is_deleted
 FROM token_transfers 
-WHERE token_type = 'erc6909'
-UNION ALL
+WHERE token_type = 'erc6909';
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS token_balances_erc6909_to_mv
+TO token_balances
+AS
 SELECT
   chain_id,
   token_type,