Skip to content

Commit 1a1ca22

Browse files
jakeloojakeloo
committed
Initial commit of new token mvs
1 parent 0381b98 commit 1a1ca22

6 files changed

+448
-2
lines changed

‎internal/tools/clickhouse/0010_clickhouse_create_token_transfers_mv.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ CREATE TABLE IF NOT EXISTS token_transfers
1414
`sign` Int8 DEFAULT 1,
1515
`insert_timestamp` DateTime DEFAULT now(),
1616

17-
INDEX minmax_block_number block_number TYPE minmax GRANULARITY 16,
18-
INDEX minmax_block_timestamp block_timestamp TYPE minmax GRANULARITY 16,
17+
INDEX minmax_block_number block_number TYPE minmax GRANULARITY 4,
18+
INDEX minmax_block_timestamp block_timestamp TYPE minmax GRANULARITY 4,
1919

2020
PROJECTION from_address_projection
2121
(

‎internal/tools/clickhouse/0011_clickhouse_create_logs_transfers_erc20_mv.sql‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
---- Setup the tables for logs transfers
2+
CREATE TABLE IF NOT EXISTS logs_transfers_erc20
3+
(
4+
`chain_id` UInt256,
5+
`token_address` FixedString(42),
6+
`from_address` FixedString(42),
7+
`to_address` FixedString(42),
8+
`block_number` UInt256,
9+
`block_timestamp` DateTime CODEC(Delta(4), ZSTD(1)),
10+
`transaction_hash` FixedString(66),
11+
`amount` UInt256,
12+
`log_index` UInt64,
13+
`sign` Int8 DEFAULT 1,
14+
`insert_timestamp` DateTime DEFAULT now(),
15+
16+
INDEX minmax_block_number block_number TYPE minmax GRANULARITY 4,
17+
INDEX minmax_block_timestamp block_timestamp TYPE minmax GRANULARITY 4,
18+
INDEX bloomfilter_token_address token_address TYPE bloom_filter GRANULARITY 4,
19+
INDEX bloomfilter_from_address from_address TYPE bloom_filter GRANULARITY 4,
20+
INDEX bloomfilter_to_address to_address TYPE bloom_filter GRANULARITY 4,
21+
INDEX bloomfilter_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 4,
22+
)
23+
ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
24+
PARTITION BY (chain_id, toStartOfYear(block_timestamp))
25+
ORDER BY (chain_id, token_address, block_number, transaction_hash, log_index)
26+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';
27+
28+
29+
30+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_to_erc20
31+
TO logs_transfers_erc20
32+
AS
33+
SELECT
34+
chain_id,
35+
address AS token_address,
36+
concat('0x', substring(topic_1, 27, 40)) AS from_address,
37+
concat('0x', substring(topic_2, 27, 40)) AS to_address,
38+
block_number,
39+
block_timestamp,
40+
transaction_hash,
41+
reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))) AS amount,
42+
log_index,
43+
sign,
44+
insert_timestamp
45+
FROM logs
46+
WHERE topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -- Transfer
47+
AND topic_3 = ''
48+
AND length(data) = 66;

‎internal/tools/clickhouse/0012_clickhouse_create_logs_transfers_erc721_mv.sql‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
CREATE TABLE IF NOT EXISTS logs_transfers_erc721
2+
(
3+
`chain_id` UInt256,
4+
`token_address` FixedString(42),
5+
`token_id` UInt256,
6+
`from_address` FixedString(42),
7+
`to_address` FixedString(42),
8+
`block_number` UInt256,
9+
`block_timestamp` DateTime CODEC(Delta(4), ZSTD(1)),
10+
`transaction_hash` FixedString(66),
11+
`amount` UInt8 DEFAULT 1,
12+
`log_index` UInt64,
13+
`sign` Int8 DEFAULT 1,
14+
`insert_timestamp` DateTime DEFAULT now(),
15+
16+
INDEX minmax_block_number block_number TYPE minmax GRANULARITY 4,
17+
INDEX minmax_block_timestamp block_timestamp TYPE minmax GRANULARITY 4,
18+
INDEX bloomfilter_token_address token_address TYPE bloom_filter GRANULARITY 4,
19+
INDEX bloomfilter_token_id token_id TYPE bloom_filter GRANULARITY 4,
20+
INDEX bloomfilter_from_address from_address TYPE bloom_filter GRANULARITY 4,
21+
INDEX bloomfilter_to_address to_address TYPE bloom_filter GRANULARITY 4,
22+
INDEX bloomfilter_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 4,
23+
)
24+
ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
25+
PARTITION BY (chain_id, toStartOfYear(block_timestamp))
26+
ORDER BY (chain_id, token_address, block_number, transaction_hash, log_index)
27+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';
28+
29+
30+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_to_erc721
31+
TO logs_transfers_erc721
32+
AS
33+
SELECT
34+
chain_id,
35+
address AS token_address,
36+
reinterpretAsUInt256(reverse(unhex(substring(topic_3, 3, 64)))) AS token_id,
37+
concat('0x', substring(topic_1, 27, 40)) AS from_address,
38+
concat('0x', substring(topic_2, 27, 40)) AS to_address,
39+
block_number,
40+
block_timestamp,
41+
transaction_hash,
42+
toUInt8(1) AS amount,
43+
log_index,
44+
sign,
45+
insert_timestamp
46+
FROM logs
47+
WHERE topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
48+
AND topic_3 != ''
49+
AND length(topic_3) = 66
50+
AND length(data) = 2;

‎internal/tools/clickhouse/0013_clickhouse_create_logs_transfers_erc1155_mv.sql‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
CREATE TABLE IF NOT EXISTS logs_transfers_erc1155
2+
(
3+
`chain_id` UInt256,
4+
`token_address` FixedString(42),
5+
`token_id` UInt256,
6+
`from_address` FixedString(42),
7+
`to_address` FixedString(42),
8+
`block_number` UInt256,
9+
`block_timestamp` DateTime CODEC(Delta(4), ZSTD(1)),
10+
`transaction_hash` FixedString(66),
11+
`amount` UInt256,
12+
`log_index` UInt64,
13+
`batch_index` UInt16 DEFAULT 0,
14+
`sign` Int8 DEFAULT 1,
15+
`insert_timestamp` DateTime DEFAULT now(),
16+
17+
INDEX minmax_block_number block_number TYPE minmax GRANULARITY 4,
18+
INDEX minmax_block_timestamp block_timestamp TYPE minmax GRANULARITY 4,
19+
INDEX bloomfilter_token_address token_address TYPE bloom_filter GRANULARITY 4,
20+
INDEX bloomfilter_token_id token_id TYPE bloom_filter GRANULARITY 4,
21+
INDEX bloomfilter_from_address from_address TYPE bloom_filter GRANULARITY 4,
22+
INDEX bloomfilter_to_address to_address TYPE bloom_filter GRANULARITY 4,
23+
INDEX bloomfilter_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 4,
24+
)
25+
ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
26+
PARTITION BY (chain_id, toStartOfYear(block_timestamp))
27+
ORDER BY (chain_id, token_address, block_number, transaction_hash, log_index, batch_index)
28+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';
29+
30+
31+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_to_erc1155_all
32+
TO logs_transfers_erc1155
33+
AS
34+
35+
SELECT
36+
chain_id,
37+
address AS token_address,
38+
reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64)))) AS token_id,
39+
concat('0x', substring(topic_2, 27, 40)) AS from_address,
40+
concat('0x', substring(topic_3, 27, 40)) AS to_address,
41+
block_number,
42+
block_timestamp,
43+
transaction_hash AS transaction_hash,
44+
reinterpretAsUInt256(reverse(unhex(substring(data, 67, 64)))) AS amount,
45+
log_index,
46+
toUInt16(0) AS batch_index,
47+
sign,
48+
insert_timestamp
49+
FROM logs
50+
WHERE topic_0 = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62' -- TransferSingle
51+
AND length(topic_2) = 66 AND length(topic_3) = 66
52+
AND length(data) = (2 + 2*64) -- 0x + 2 words = 130
53+
54+
UNION ALL
55+
56+
WITH meta AS
57+
(
58+
SELECT
59+
chain_id, address, topic_2, topic_3, data,
60+
block_number, block_timestamp, transaction_hash, log_index, sign, insert_timestamp,
61+
toUInt32(3 + 2*64) AS ids_len_idx, -- "0x" + 2*32B heads
62+
reinterpretAsUInt64(reverse(unhex(substring(data, ids_len_idx, 64)))) AS ids_len,
63+
(ids_len_idx + 64) AS ids_vals_idx,
64+
(ids_len_idx + 64) + ids_len * 64 AS amts_len_idx,
65+
reinterpretAsUInt64(reverse(unhex(substring(data, amts_len_idx, 64)))) AS amts_len,
66+
(amts_len_idx + 64) AS amts_vals_idx,
67+
(2 + (4 + ids_len + amts_len) * 64) AS expected_len
68+
FROM logs
69+
WHERE topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb' -- TransferBatch
70+
AND length(topic_2) = 66 AND length(topic_3) = 66
71+
),
72+
expanded AS
73+
(
74+
SELECT
75+
*,
76+
arrayMap(i -> substring(data, ids_vals_idx + (i-1)*64, 64), range(1, toInt32(ids_len) + 1)) AS ids_hex,
77+
arrayMap(i -> substring(data, amts_vals_idx + (i-1)*64, 64), range(1, toInt32(amts_len) + 1)) AS amts_hex
78+
FROM meta
79+
WHERE amts_len = ids_len
80+
AND length(data) = expected_len
81+
)
82+
SELECT
83+
chain_id,
84+
address AS token_address,
85+
reinterpretAsUInt256(reverse(unhex(id_hex))) AS token_id,
86+
concat('0x', substring(topic_2, 27, 40)) AS from_address,
87+
concat('0x', substring(topic_3, 27, 40)) AS to_address,
88+
block_number,
89+
block_timestamp,
90+
transaction_hash AS transaction_hash,
91+
reinterpretAsUInt256(reverse(unhex(amt_hex))) AS amount,
92+
log_index,
93+
toUInt16(idx - 1) AS batch_index, -- make it 0-based
94+
sign,
95+
insert_timestamp
96+
FROM expanded
97+
ARRAY JOIN
98+
ids_hex AS id_hex,
99+
amts_hex AS amt_hex,
100+
arrayEnumerate(ids_hex) AS idx;

‎internal/tools/clickhouse/0014_clickhouse_create_logs_transfers_erc6909_mv.sql‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
CREATE TABLE IF NOT EXISTS logs_transfers_erc6909
2+
(
3+
`chain_id` UInt256,
4+
`token_address` FixedString(42),
5+
`token_id` UInt256,
6+
`from_address` FixedString(42),
7+
`to_address` FixedString(42),
8+
`block_number` UInt256,
9+
`block_timestamp` DateTime CODEC(Delta(4), ZSTD(1)),
10+
`transaction_hash` FixedString(66),
11+
`amount` UInt256,
12+
`log_index` UInt64,
13+
`sign` Int8 DEFAULT 1,
14+
`insert_timestamp` DateTime DEFAULT now(),
15+
16+
INDEX minmax_block_number block_number TYPE minmax GRANULARITY 4,
17+
INDEX minmax_block_timestamp block_timestamp TYPE minmax GRANULARITY 4,
18+
INDEX bloomfilter_token_address token_address TYPE bloom_filter GRANULARITY 4,
19+
INDEX bloomfilter_token_id token_id TYPE bloom_filter GRANULARITY 4,
20+
INDEX bloomfilter_from_address from_address TYPE bloom_filter GRANULARITY 4,
21+
INDEX bloomfilter_to_address to_address TYPE bloom_filter GRANULARITY 4,
22+
INDEX bloomfilter_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 4,
23+
)
24+
ENGINE = VersionedCollapsingMergeTree(sign, insert_timestamp)
25+
PARTITION BY (chain_id, toStartOfYear(block_timestamp))
26+
ORDER BY (chain_id, token_address, block_number, transaction_hash, log_index)
27+
SETTINGS index_granularity = 8192, lightweight_mutation_projection_mode = 'rebuild', deduplicate_merge_projection_mode = 'rebuild';
28+
29+
30+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_to_erc6909
31+
TO logs_transfers_erc6909
32+
AS
33+
SELECT
34+
chain_id,
35+
lower(address) AS token_address,
36+
reinterpretAsUInt256(reverse(unhex(substring(topic_3, 3, 64)))) AS token_id,
37+
lower(concat('0x', substring(topic_1, 27, 40))) AS from_address,
38+
lower(concat('0x', substring(topic_2, 27, 40))) AS to_address,
39+
block_number,
40+
block_timestamp,
41+
transaction_hash,
42+
reinterpretAsUInt256(reverse(unhex(substring(data, 67, 64)))) AS amount,
43+
log_index,
44+
sign,
45+
insert_timestamp
46+
FROM logs
47+
WHERE topic_0 = '0x1b3d7edb2e9c0b0e7c525b20aaaef0f5940d2ed71663c7d39266ecafac728859'
48+
AND length(topic_1) = 66
49+
AND length(topic_2) = 66
50+
AND length(data) == 2 + 128;

0 commit comments

Comments
 (0)