Skip to content

Commit 136a346

Browse files
jakeloojakeloo
committed
redis tls. erc1155 batch mv
1 parent 4595fa6 commit 136a346

File tree

4 files changed

+64
-48
lines changed

4 files changed

+64
-48
lines changed

‎cmd/root.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ func init() {
151151
rootCmd.PersistentFlags().Int("storage-orchestrator-redis-port", 6379, "Redis port for orchestrator storage metadata")
152152
rootCmd.PersistentFlags().String("storage-orchestrator-redis-password", "", "Redis password for orchestator storage metadata")
153153
rootCmd.PersistentFlags().Int("storage-orchestrator-redis-db", 0, "Redis database number for orchestrator storage metadata")
154+
rootCmd.PersistentFlags().Bool("storage-orchestrator-redis-enableTLS", true, "Enable TLS for Redis connection in orchestrator storage metadata")
154155
rootCmd.PersistentFlags().String("storage-staging-type", "auto", "Storage type for staging (auto, clickhouse, postgres, kafka, badger, s3)")
155156
rootCmd.PersistentFlags().String("storage-main-type", "auto", "Storage type for main (auto, clickhouse, postgres, kafka, badger, s3)")
156157
rootCmd.PersistentFlags().String("storage-orchestrator-type", "auto", "Storage type for orchestrator (auto, clickhouse, postgres, badger)")
@@ -341,6 +342,7 @@ func init() {
341342
viper.BindPFlag("storage.orchestrator.redis.port", rootCmd.PersistentFlags().Lookup("storage-orchestrator-redis-port"))
342343
viper.BindPFlag("storage.orchestrator.redis.password", rootCmd.PersistentFlags().Lookup("storage-orchestrator-redis-password"))
343344
viper.BindPFlag("storage.orchestrator.redis.db", rootCmd.PersistentFlags().Lookup("storage-orchestrator-redis-db"))
345+
viper.BindPFlag("storage.orchestrator.redis.enableTLS", rootCmd.PersistentFlags().Lookup("storage-orchestrator-redis-enableTLS"))
344346
viper.BindPFlag("storage.orchestrator.badger.path", rootCmd.PersistentFlags().Lookup("storage-orchestrator-badger-path"))
345347
viper.BindPFlag("storage.orchestrator.type", rootCmd.PersistentFlags().Lookup("storage-orchestrator-type"))
346348
viper.BindPFlag("storage.staging.postgres.host", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-host"))

‎configs/config.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,10 +147,11 @@ type PostgresConfig struct {
147147
}
148148

149149
type RedisConfig struct {
150-
Host string `mapstructure:"host"`
151-
Port int `mapstructure:"port"`
152-
Password string `mapstructure:"password"`
153-
DB int `mapstructure:"db"`
150+
Host string `mapstructure:"host"`
151+
Port int `mapstructure:"port"`
152+
Password string `mapstructure:"password"`
153+
DB int `mapstructure:"db"`
154+
EnableTLS bool `mapstructure:"enableTLS"`
154155
}
155156

156157
type KafkaConfig struct {

‎internal/storage/redis.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package storage
22

33
import (
44
"context"
5+
"crypto/tls"
56
"fmt"
67
"math/big"
78
"time"
@@ -26,10 +27,18 @@ type RedisConnector struct {
2627

2728
func NewRedisConnector(cfg *config.RedisConfig) (*RedisConnector, error) {
2829
// Connect to Redis
30+
var tlsConfig *tls.Config
31+
if cfg.EnableTLS {
32+
tlsConfig = &tls.Config{
33+
MinVersion: tls.VersionTLS12, // Ensure a secure TLS version
34+
}
35+
}
36+
2937
redisClient := redis.NewClient(&redis.Options{
30-
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
31-
Password: cfg.Password,
32-
DB: cfg.DB,
38+
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
39+
Password: cfg.Password,
40+
DB: cfg.DB,
41+
TLSConfig: tlsConfig,
3342
})
3443

3544
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

‎internal/tools/clickhouse/0007_clickhouse_create_token_transfers_mv.sql‎

Lines changed: 45 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -80,52 +80,56 @@ WHERE topic_0 = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0
8080
-- ERC1155 (batch)
8181
CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc1155_batch_mv
8282
TO token_transfers
83-
AS
84-
SELECT
85-
chain_id,
86-
address AS token_address,
87-
'erc1155' AS token_type,
88-
reinterpretAsUInt256(reverse(unhex(id_hex))) AS token_id,
89-
concat('0x', substring(topic_2, 27, 40)) AS from_address,
90-
concat('0x', substring(topic_3, 27, 40)) AS to_address,
83+
AS
84+
SELECT
85+
chain_id,
86+
address AS token_address,
87+
'erc1155' AS token_type,
88+
reinterpretAsUInt256(reverse(substring(bin, (ids_base + ((i - 1) * 32)) + 1, 32))) AS token_id,
89+
concat('0x', substring(topic_2, 27, 40)) AS from_address,
90+
concat('0x', substring(topic_3, 27, 40)) AS to_address,
91+
block_number,
92+
block_timestamp,
93+
transaction_hash,
94+
transaction_index,
95+
reinterpretAsUInt256(reverse(substring(bin, (am_base + ((i - 1) * 32)) + 1, 32))) AS amount,
96+
log_index,
97+
toNullable(toUInt16(i - 1)) AS batch_index,
98+
insert_timestamp,
99+
is_deleted
100+
FROM (
101+
SELECT
102+
chain_id,
103+
address,
104+
topic_2,
105+
topic_3,
91106
block_number,
92107
block_timestamp,
93108
transaction_hash,
94109
transaction_index,
95-
reinterpretAsUInt256(reverse(unhex(amount_hex))) AS amount,
96110
log_index,
97-
toNullable(toUInt16(array_index - 1)) AS batch_index,
98-
insert_timestamp,
99-
is_deleted
100-
FROM (
101-
SELECT
102-
chain_id,
103-
address,
104-
topic_2,
105-
topic_3,
106-
block_number,
107-
block_timestamp,
108-
transaction_hash,
109-
transaction_index,
110-
log_index,
111-
is_deleted,
112-
insert_timestamp,
113-
toUInt32(reinterpretAsUInt256(reverse(unhex(substring(data, 3, 64))))) AS ids_offset,
114-
toUInt32(reinterpretAsUInt256(reverse(unhex(substring(data, 67, 64))))) AS amounts_offset,
115-
toUInt32(reinterpretAsUInt256(reverse(unhex(substring(data, 3 + ids_offset * 2, 64))))) AS ids_length,
116-
toUInt32(reinterpretAsUInt256(reverse(unhex(substring(data, 3 + amounts_offset * 2, 64))))) AS amounts_length,
117-
arrayMap(i -> substring(data, 3 + ids_offset * 2 + 64 + (i-1)*64, 64), range(1, least(ids_length, 10000) + 1)) AS ids_array,
118-
arrayMap(i -> substring(data, 3 + amounts_offset * 2 + 64 + (i-1)*64, 64), range(1, least(amounts_length, 10000) + 1)) AS amounts_array
119-
FROM logs
120-
WHERE topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb'
121-
AND length(topic_2) = 66
122-
AND length(topic_3) = 66
123-
AND ids_length = amounts_length
124-
)
125-
ARRAY JOIN
126-
ids_array AS id_hex,
127-
amounts_array AS amount_hex,
128-
arrayEnumerate(ids_array) AS array_index;
111+
is_deleted,
112+
insert_timestamp,
113+
unhex(substring(data, 3)) AS bin,
114+
length(unhex(substring(data, 3))) AS bin_len,
115+
toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), 1, 32)))) AS ids_off,
116+
toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), 33, 32)))) AS am_off,
117+
toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), ids_off + 1, 32)))) AS ids_len,
118+
toUInt32(reinterpretAsUInt256(reverse(substring(unhex(substring(data, 3)), am_off + 1, 32)))) AS am_len,
119+
ids_off + 32 AS ids_base,
120+
am_off + 32 AS am_base
121+
FROM default.logs
122+
WHERE (topic_0 = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb')
123+
AND (length(topic_2) = 66)
124+
AND (length(topic_3) = 66)
125+
AND (ids_len = am_len)
126+
AND (ids_len > 0)
127+
AND ((ids_off + 32) <= bin_len)
128+
AND ((am_off + 32) <= bin_len)
129+
AND ((ids_base + (ids_len * 32)) <= bin_len)
130+
AND ((am_base + (am_len * 32)) <= bin_len)
131+
) ARRAY JOIN range(1, ids_len + 1) AS i;
132+
129133

130134
-- ERC6909
131135
CREATE MATERIALIZED VIEW IF NOT EXISTS token_transfers_erc6909_mv

0 commit comments

Comments
 (0)