fix projections, use _part_offset projections

Author: jakeloo

internal/tools/clickhouse/0000_clickhouse_create_blocks_table.sql

@@ -27,6 +27,22 @@ CREATE TABLE IF NOT EXISTS blocks (
 
     INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
     INDEX idx_hash hash TYPE bloom_filter GRANULARITY 2,
+
+    PROJECTION chain_state_projection
+    (
+        SELECT
+          chain_id,
+          count() AS count,
+          uniqExact(block_number) AS unique_block_count,
+          min(block_number) AS min_block_number,
+          min(block_timestamp) AS min_block_timestamp,
+          max(block_number) AS max_block_number,
+          max(block_timestamp) AS max_block_timestamp
+        GROUP BY
+          chain_id
+    )
+
+
 ) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
 ORDER BY (chain_id, block_number)
 PARTITION BY (chain_id, toStartOfQuarter(block_timestamp))

internal/tools/clickhouse/0001_clickhouse_create_transactions_table.sql

@@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS transactions (
     PROJECTION from_address_projection
     (
         SELECT
-          *
+          _part_offset
         ORDER BY 
           chain_id,
           from_address,
@@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS transactions (
     PROJECTION to_address_projection
     (
         SELECT
-          *
+          _part_offset
         ORDER BY
           chain_id,
           to_address,
@@ -67,11 +67,12 @@ CREATE TABLE IF NOT EXISTS transactions (
         SELECT
           chain_id,
           from_address,
-          countState() AS tx_count_state,
-          minState(block_number) AS min_block_number_state,
-          minState(block_timestamp) AS min_block_timestamp_state,
-          maxState(block_number) AS max_block_number_state,
-          maxState(block_timestamp) AS max_block_timestamp_state
+          count() AS tx_count,
+          uniqExact(hash) AS unique_tx_count,
+          min(block_number) AS min_block_number,
+          min(block_timestamp) AS min_block_timestamp,
+          max(block_number) AS max_block_number,
+          max(block_timestamp) AS max_block_timestamp
         GROUP BY
           chain_id,
           from_address
@@ -81,16 +82,17 @@ CREATE TABLE IF NOT EXISTS transactions (
         SELECT
           chain_id,
           to_address,
-          countState() AS tx_count_state,
-          minState(block_number) AS min_block_number_state,
-          minState(block_timestamp) AS min_block_timestamp_state,
-          maxState(block_number) AS max_block_number_state,
-          maxState(block_timestamp) AS max_block_timestamp_state
+          count() AS tx_count,
+          uniqExact(hash) AS unique_tx_count,
+          min(block_number) AS min_block_number,
+          min(block_timestamp) AS min_block_timestamp,
+          max(block_number) AS max_block_number,
+          max(block_timestamp) AS max_block_timestamp
         GROUP BY
           chain_id,
           to_address
     )
 ) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
 ORDER BY (chain_id, block_number, hash)
 PARTITION BY (chain_id, toStartOfQuarter(block_timestamp))
-SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild';
+SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild', allow_part_offset_column_in_projections=1;

internal/tools/clickhouse/0002_clickhouse_create_logs_table.sql

@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS logs (
     PROJECTION chain_address_topic0_projection
     (
         SELECT
-            *
+            _part_offset
         ORDER BY 
             chain_id,
             address,
@@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS logs (
     PROJECTION chain_topic0_projection
     (
         SELECT
-            *
+            _part_offset
         ORDER BY 
             chain_id,
             topic_0,
@@ -55,11 +55,11 @@ CREATE TABLE IF NOT EXISTS logs (
             chain_id,
             address,
             topic_0,
-            countState() AS log_count_state,
-            minState(block_number) AS min_block_number_state,
-            minState(block_timestamp) AS min_block_timestamp_state,
-            maxState(block_number) AS max_block_number_state,
-            maxState(block_timestamp) AS max_block_timestamp_state
+            count() AS log_count,
+            min(block_number) AS min_block_number,
+            min(block_timestamp) AS min_block_timestamp,
+            max(block_number) AS max_block_number,
+            max(block_timestamp) AS max_block_timestamp
         GROUP BY
             chain_id,
             address,
@@ -68,4 +68,4 @@ CREATE TABLE IF NOT EXISTS logs (
 ) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
 ORDER BY (chain_id, block_number, transaction_hash, log_index)
 PARTITION BY (chain_id, toStartOfQuarter(block_timestamp))
-SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild';
+SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild', allow_part_offset_column_in_projections=1;

internal/tools/clickhouse/0003_clickhouse_create_traces_table.sql

@@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS traces (
     PROJECTION from_address_projection
     (
         SELECT
-          *
+          _part_offset
         ORDER BY 
           chain_id,
           from_address,
@@ -43,7 +43,7 @@ CREATE TABLE IF NOT EXISTS traces (
     PROJECTION to_address_projection
     (
         SELECT
-          *
+          _part_offset
         ORDER BY 
           chain_id,
           to_address,
@@ -55,4 +55,4 @@ CREATE TABLE IF NOT EXISTS traces (
 ) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
 ORDER BY (chain_id, transaction_hash, trace_address)
 PARTITION BY (chain_id, toStartOfQuarter(block_timestamp))
-SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild';
+SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild', allow_part_offset_column_in_projections=1;

internal/tools/clickhouse/0006_clickhouse_create_token_transfers.sql

@@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS token_transfers
 
     PROJECTION from_address_projection (
         SELECT
-            *
+            _part_offset
         ORDER BY
             chain_id,
             from_address,
@@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS token_transfers
     ),
     PROJECTION to_address_projection (
         SELECT
-            *
+            _part_offset
         ORDER BY
             chain_id,
             to_address,
@@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS token_transfers
     ),
     PROJECTION token_id_projection (
         SELECT 
-            *
+            _part_offset
         ORDER BY
             chain_id,
             token_address,
@@ -59,12 +59,12 @@ CREATE TABLE IF NOT EXISTS token_transfers
             from_address,
             token_address,
             token_type,
-            countState() AS transfer_count_state,
-            sumState(toInt256(amount)) AS total_amount_state,
-            minState(block_number) AS min_block_number_state,
-            minState(block_timestamp) AS min_block_timestamp_state,
-            maxState(block_number) AS max_block_number_state,
-            maxState(block_timestamp) AS max_block_timestamp_state
+            count() AS transfer_count,
+            sum(toInt256(amount)) AS total_amount,
+            min(block_number) AS min_block_number,
+            min(block_timestamp) AS min_block_timestamp,
+            max(block_number) AS max_block_number,
+            max(block_timestamp) AS max_block_timestamp
         GROUP BY
             chain_id,
             from_address,
@@ -77,12 +77,12 @@ CREATE TABLE IF NOT EXISTS token_transfers
             to_address,
             token_address,
             token_type,
-            countState() AS transfer_count_state,
-            sumState(toInt256(amount)) AS total_amount_state,
-            minState(block_number) AS min_block_number_state,
-            minState(block_timestamp) AS min_block_timestamp_state,
-            maxState(block_number) AS max_block_number_state,
-            maxState(block_timestamp) AS max_block_timestamp_state
+            count() AS transfer_count,
+            sum(toInt256(amount)) AS total_amount,
+            min(block_number) AS min_block_number,
+            min(block_timestamp) AS min_block_timestamp,
+            max(block_number) AS max_block_number,
+            max(block_timestamp) AS max_block_timestamp
         GROUP BY
             chain_id,
             to_address,
@@ -95,12 +95,12 @@ CREATE TABLE IF NOT EXISTS token_transfers
             token_address,
             token_id,
             token_type,
-            countState() AS transfer_count_state,
-            sumState(toInt256(amount)) AS total_volume_state,
-            minState(block_number) AS min_block_number_state,
-            minState(block_timestamp) AS min_block_timestamp_state,
-            maxState(block_number) AS max_block_number_state,
-            maxState(block_timestamp) AS max_block_timestamp_state
+            count() AS transfer_count,
+            sum(toInt256(amount)) AS total_volume,
+            min(block_number) AS min_block_number,
+            min(block_timestamp) AS min_block_timestamp,
+            max(block_number) AS max_block_number,
+            max(block_timestamp) AS max_block_timestamp
         GROUP BY
             chain_id,
             token_address,
@@ -111,4 +111,4 @@ CREATE TABLE IF NOT EXISTS token_transfers
 ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
 PARTITION BY (chain_id, toStartOfQuarter(block_timestamp))
 ORDER BY (chain_id, token_address, block_number, transaction_index, log_index)
-SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';
+SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild', allow_part_offset_column_in_projections=1;

internal/tools/clickhouse/0008_clickhouse_create_token_balances.sql

@@ -52,9 +52,16 @@ CREATE TABLE IF NOT EXISTS token_balances
       maxState(block_number) AS max_block_number_state,
       maxState(block_timestamp) AS max_block_timestamp_state
     GROUP BY chain_id, token_address, token_id, owner_address
+  ),
+
+  PROJECTION token_projection
+  (
+    SELECT
+      _part_offset
+    ORDER BY chain_id, token_address, token_id, owner_address
   )
 )
 ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted)
 PARTITION BY chain_id
 ORDER BY (chain_id, owner_address, token_address, token_id, block_number, transaction_index, log_index, direction)
-SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';
+SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild', allow_part_offset_column_in_projections=1;

internal/tools/clickhouse/0010_clickhouse_create_address_transactions.sql

@@ -47,11 +47,12 @@ CREATE TABLE IF NOT EXISTS address_transactions (
         SELECT
           chain_id,
           address,
-          countState() AS tx_count_state,
-          minState(block_number) AS min_block_number_state,
-          minState(block_timestamp) AS min_block_timestamp_state,
-          maxState(block_number) AS max_block_number_state,
-          maxState(block_timestamp) AS max_block_timestamp_state
+          count() AS tx_count,
+          uniqExact(hash) AS unique_tx_count,
+          min(block_number) AS min_block_number,
+          min(block_timestamp) AS min_block_timestamp,
+          max(block_number) AS max_block_number,
+          max(block_timestamp) AS max_block_timestamp
         GROUP BY
           chain_id,
           address

internal/tools/clickhouse/0012_clickhouse_create_address_transfers.sql

@@ -30,12 +30,12 @@ CREATE TABLE IF NOT EXISTS address_transfers (
             address_type,
             token_address,
             token_type,
-            countState() AS transfer_count_state,
-            sumState(toInt256(amount)) AS total_amount_state,
-            minState(block_number) AS min_block_number_state,
-            minState(block_timestamp) AS min_block_timestamp_state,
-            maxState(block_number) AS max_block_number_state,
-            maxState(block_timestamp) AS max_block_timestamp_state
+            count() AS transfer_count,
+            sum(toInt256(amount)) AS total_amount,
+            min(block_number) AS min_block_number,
+            min(block_timestamp) AS min_block_timestamp,
+            max(block_number) AS max_block_number,
+            max(block_timestamp) AS max_block_timestamp
         GROUP BY
             chain_id,
             address,
@@ -49,12 +49,12 @@ CREATE TABLE IF NOT EXISTS address_transfers (
             address,
             token_address,
             token_type,
-            countState() AS transfer_count_state,
-            sumState(toInt256(amount)) AS total_amount_state,
-            minState(block_number) AS min_block_number_state,
-            minState(block_timestamp) AS min_block_timestamp_state,
-            maxState(block_number) AS max_block_number_state,
-            maxState(block_timestamp) AS max_block_timestamp_state
+            count() AS transfer_count,
+            sum(toInt256(amount)) AS total_amount,
+            min(block_number) AS min_block_number,
+            min(block_timestamp) AS min_block_timestamp,
+            max(block_number) AS max_block_number,
+            max(block_timestamp) AS max_block_timestamp
         GROUP BY
             chain_id,
             address,