Skip to content

Commit 661b150

Browse files
jakeloojakeloo
committed
Update queries
1 parent 4775d57 commit 661b150

15 files changed

+522
-408
lines changed

‎internal/tools/clickhouse/0004_clickhouse_create_blocks_table.sql‎ renamed to ‎internal/tools/clickhouse/0000_clickhouse_create_blocks_table.sql‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ CREATE TABLE IF NOT EXISTS blocks (
2121
`gas_used` UInt256,
2222
`withdrawals_root` FixedString(66),
2323
`base_fee_per_gas` Nullable(UInt64),
24+
2425
`insert_timestamp` DateTime DEFAULT now(),
2526
`sign` Int8 DEFAULT 1,
26-
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 3,
27-
INDEX idx_hash hash TYPE bloom_filter GRANULARITY 3,
27+
28+
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
29+
INDEX idx_hash hash TYPE bloom_filter GRANULARITY 2,
2830
) ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
2931
ORDER BY (chain_id, block_number)
30-
PARTITION BY chain_id;
32+
PARTITION BY toYYYYMM(block_timestamp);

‎internal/tools/clickhouse/0001_clickhouse_create_block_failures_table.sql‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎internal/tools/clickhouse/0005_clickhouse_create_transactions_table.sql‎ renamed to ‎internal/tools/clickhouse/0001_clickhouse_create_transactions_table.sql‎

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,32 +31,52 @@ CREATE TABLE IF NOT EXISTS transactions (
3131
`blob_gas_price` Nullable(UInt256),
3232
`logs_bloom` Nullable(String),
3333
`status` Nullable(UInt64),
34+
3435
`sign` Int8 DEFAULT 1,
3536
`insert_timestamp` DateTime DEFAULT now(),
36-
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 3,
37+
38+
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
3739
INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 3,
38-
INDEX idx_hash hash TYPE bloom_filter GRANULARITY 3,
39-
INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1,
40-
INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1,
41-
INDEX idx_function_selector function_selector TYPE bloom_filter GRANULARITY 1,
42-
PROJECTION txs_chainid_from_address
40+
INDEX idx_hash hash TYPE bloom_filter GRANULARITY 2,
41+
INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 4,
42+
INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 4,
43+
INDEX idx_function_selector function_selector TYPE bloom_filter GRANULARITY 2,
44+
45+
PROJECTION from_address_projection
4346
(
44-
SELECT *
47+
SELECT
48+
chain_id,
49+
block_number,
50+
block_timestamp,
51+
hash,
52+
from_address,
53+
to_address,
54+
value,
55+
data
4556
ORDER BY
4657
chain_id,
4758
from_address,
48-
block_number
59+
block_number,
60+
hash
4961
),
50-
PROJECTION txs_chainid_to_address
62+
PROJECTION to_address_projection
5163
(
52-
SELECT *
53-
ORDER BY
64+
SELECT
65+
chain_id,
66+
block_number,
67+
block_timestamp,
68+
hash,
69+
from_address,
70+
to_address,
71+
value,
72+
data
73+
ORDER BY
5474
chain_id,
5575
to_address,
5676
block_number,
5777
hash
5878
)
5979
) ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
6080
ORDER BY (chain_id, block_number, hash)
61-
PARTITION BY chain_id
62-
SETTINGS deduplicate_merge_projection_mode = 'drop', lightweight_mutation_projection_mode = 'rebuild';
81+
PARTITION BY toYYYYMM(block_timestamp)
82+
SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild';

‎internal/tools/clickhouse/0002_clickhouse_create_cursors_table.sql‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎internal/tools/clickhouse/0006_clickhouse_create_logs_table.sql‎ renamed to ‎internal/tools/clickhouse/0002_clickhouse_create_logs_table.sql‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,28 +12,44 @@ CREATE TABLE IF NOT EXISTS logs (
1212
`topic_1` String,
1313
`topic_2` String,
1414
`topic_3` String,
15-
`insert_timestamp` DateTime DEFAULT now(),
15+
1616
`sign` Int8 DEFAULT 1,
17-
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 3,
17+
`insert_timestamp` DateTime DEFAULT now(),
18+
19+
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
1820
INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 3,
19-
INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 3,
20-
INDEX idx_address address TYPE bloom_filter GRANULARITY 1,
21-
INDEX idx_topic0 topic_0 TYPE bloom_filter GRANULARITY 1,
22-
INDEX idx_topic1 topic_1 TYPE bloom_filter GRANULARITY 1,
23-
INDEX idx_topic2 topic_2 TYPE bloom_filter GRANULARITY 1,
24-
INDEX idx_topic3 topic_3 TYPE bloom_filter GRANULARITY 1,
25-
PROJECTION logs_chainid_topic0_address
21+
INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 2,
22+
INDEX idx_address address TYPE bloom_filter GRANULARITY 3,
23+
INDEX idx_topic0 topic_0 TYPE bloom_filter GRANULARITY 3,
24+
INDEX idx_topic1 topic_1 TYPE bloom_filter GRANULARITY 4,
25+
INDEX idx_topic2 topic_2 TYPE bloom_filter GRANULARITY 4,
26+
INDEX idx_topic3 topic_3 TYPE bloom_filter GRANULARITY 4,
27+
28+
PROJECTION chain_address_topic0_projection
2629
(
27-
SELECT *
30+
SELECT
31+
*
2832
ORDER BY
2933
chain_id,
30-
topic_0,
3134
address,
35+
topic_0,
3236
block_number,
3337
transaction_index,
3438
log_index
39+
),
40+
PROJECTION chain_topic0_projection
41+
(
42+
SELECT
43+
*
44+
ORDER BY
45+
chain_id,
46+
topic_0,
47+
block_number,
48+
transaction_index,
49+
log_index,
50+
address
3551
)
3652
) ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
3753
ORDER BY (chain_id, block_number, transaction_hash, log_index)
38-
PARTITION BY chain_id
39-
SETTINGS deduplicate_merge_projection_mode = 'drop', lightweight_mutation_projection_mode = 'rebuild';
54+
PARTITION BY toYYYYMM(block_timestamp)
55+
SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild';

‎internal/tools/clickhouse/0003_clickhouse_create_staging_table.sql‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎internal/tools/clickhouse/0007_clickhouse_create_traces_table.sql‎ renamed to ‎internal/tools/clickhouse/0003_clickhouse_create_traces_table.sql‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,39 @@ CREATE TABLE IF NOT EXISTS traces (
2020
`author` Nullable(FixedString(42)),
2121
`reward_type` LowCardinality(Nullable(String)),
2222
`refund_address` Nullable(FixedString(42)),
23+
2324
`sign` Int8 DEFAULT 1,
2425
`insert_timestamp` DateTime DEFAULT now(),
25-
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 3,
26-
INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 3,
27-
INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 3,
28-
INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1,
29-
INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1,
26+
27+
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
28+
INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 2,
29+
INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 3,
30+
INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 3,
31+
32+
PROJECTION from_address_projection
33+
(
34+
SELECT
35+
*
36+
ORDER BY
37+
chain_id,
38+
from_address,
39+
block_number,
40+
transaction_hash,
41+
trace_address
42+
),
43+
PROJECTION to_address_projection
44+
(
45+
SELECT
46+
*
47+
ORDER BY
48+
chain_id,
49+
to_address,
50+
block_number,
51+
transaction_hash,
52+
trace_address
53+
)
54+
3055
) ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
31-
ORDER BY (chain_id, block_number, transaction_hash, trace_address)
32-
PARTITION BY chain_id;
56+
ORDER BY (chain_id, transaction_hash, trace_address)
57+
PARTITION BY toYYYYMM(block_timestamp)
58+
SETTINGS deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild';

‎internal/tools/clickhouse/0000_clickhouse_create_insert_null_table.sql‎ renamed to ‎internal/tools/clickhouse/0004_clickhouse_create_insert_null_table.sql‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
CREATE TABLE IF NOT EXISTS inserts_null_table (
1+
CREATE TABLE IF NOT EXISTS insert_null_block_data (
22
chain_id UInt256,
3+
34
block Tuple(
45
block_number UInt256,
56
block_timestamp DateTime,
@@ -92,8 +93,7 @@ CREATE TABLE IF NOT EXISTS inserts_null_table (
9293
reward_type LowCardinality(Nullable(String)),
9394
refund_address Nullable(FixedString(42))
9495
)),
95-
insert_timestamp DateTime DEFAULT now(),
96-
sign Int8 DEFAULT 1
97-
) ENGINE = MergeTree
98-
ORDER BY (chain_id, insert_timestamp)
99-
PARTITION BY chain_id;
96+
97+
sign Int8 DEFAULT 1,
98+
insert_timestamp DateTime DEFAULT now()
99+
) ENGINE = Null;

‎internal/tools/clickhouse/0008_clickhouse_create_insert_mvs.sql‎ renamed to ‎internal/tools/clickhouse/0005_clickhouse_create_insert_data_mv.sql‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_blocks_inserts
1+
CREATE MATERIALIZED VIEW IF NOT EXISTS insert_blocks_mv
22
TO blocks
33
AS
44
SELECT
@@ -26,9 +26,9 @@ SELECT
2626
block.21 AS base_fee_per_gas,
2727
insert_timestamp,
2828
sign
29-
FROM inserts_null_table;
29+
FROM insert_null_block_data;
3030

31-
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_transactions_inserts
31+
CREATE MATERIALIZED VIEW IF NOT EXISTS insert_transactions_mv
3232
TO transactions
3333
AS
3434
SELECT
@@ -66,10 +66,10 @@ SELECT
6666
t.31 AS status,
6767
insert_timestamp,
6868
sign
69-
FROM inserts_null_table
69+
FROM insert_null_block_data
7070
ARRAY JOIN transactions AS t;
7171

72-
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_inserts
72+
CREATE MATERIALIZED VIEW IF NOT EXISTS insert_logs_mv
7373
TO logs
7474
AS
7575
SELECT
@@ -88,10 +88,10 @@ SELECT
8888
l.12 AS topic_3,
8989
insert_timestamp,
9090
sign
91-
FROM inserts_null_table
91+
FROM insert_null_block_data
9292
ARRAY JOIN logs AS l;
9393

94-
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_traces_inserts
94+
CREATE MATERIALIZED VIEW IF NOT EXISTS insert_traces_mv
9595
TO traces
9696
AS
9797
SELECT
@@ -118,5 +118,5 @@ SELECT
118118
tr.20 AS refund_address,
119119
insert_timestamp,
120120
sign
121-
FROM inserts_null_table
121+
FROM insert_null_block_data
122122
ARRAY JOIN traces AS tr;

‎internal/tools/clickhouse/0006_clickhouse_create_logs_transfer.sql‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
CREATE TABLE IF NOT EXISTS logs_transfer
2+
(
3+
`chain_id` UInt256,
4+
`token_type` LowCardinality(String),
5+
`token_address` FixedString(42),
6+
`token_id` UInt256,
7+
`from_address` FixedString(42),
8+
`to_address` FixedString(42),
9+
`block_number` UInt256,
10+
`block_timestamp` DateTime CODEC(Delta(4), ZSTD(1)),
11+
`transaction_hash` FixedString(66),
12+
`transaction_index` UInt64,
13+
`amount` UInt256,
14+
`log_index` UInt64,
15+
`batch_index` Nullable(UInt16) DEFAULT NULL,
16+
17+
`sign` Int8 DEFAULT 1,
18+
`insert_timestamp` DateTime DEFAULT now(),
19+
20+
INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1,
21+
INDEX idx_token_address token_address TYPE bloom_filter GRANULARITY 2,
22+
INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 3,
23+
INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 3,
24+
INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 4,
25+
26+
PROJECTION from_address_projection (
27+
SELECT
28+
*
29+
ORDER BY
30+
chain_id,
31+
from_address,
32+
block_number,
33+
transaction_index,
34+
log_index
35+
),
36+
PROJECTION to_address_projection (
37+
SELECT
38+
*
39+
ORDER BY
40+
chain_id,
41+
to_address,
42+
block_number,
43+
transaction_index,
44+
log_index
45+
46+
),
47+
PROJECTION token_id_projection (
48+
SELECT
49+
*
50+
ORDER BY
51+
chain_id,
52+
token_address,
53+
token_id,
54+
block_number,
55+
transaction_index,
56+
log_index
57+
)
58+
)
59+
ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
60+
PARTITION BY toYYYYMM(block_timestamp)
61+
ORDER BY (chain_id, token_address, block_number, transaction_index, log_index)
62+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';

0 commit comments

Comments
 (0)