Skip to content

Commit 685bb26

Browse files
jakeloojakeloo
committed
Initial support for Postgres for orchestrator and staging
1 parent f725367 commit 685bb26

22 files changed

+733
-9
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,6 @@ cmd/api/__debug_bin*
3030

3131
configs/secrets*
3232
!configs/secrets.example.yml
33+
34+
indexer
35+
main

‎README.md‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ cp configs/secrets.example.yml configs/secrets.yml
1919
docker-compose up -d clickhouse
2020

2121
# 4. Apply ClickHouse migrations
22-
cat internal/tools/*.sql | docker exec -i <clickhouse-container> clickhouse-client --user admin --password password
22+
cat internal/tools/clickhouse/*.sql | docker exec -i <clickhouse-container> clickhouse-client --user admin --password password
23+
24+
# 4b. Apply Postgres migrations (if using Postgres for orchestration and staging)
25+
psql -h localhost -U postgres -d postgres -f internal/tools/postgres/postgres_schema.sql
2326

2427
# 5. Build and run Insight
2528
go build -o main -tags=production
@@ -438,7 +441,3 @@ insight/
438441
---
439442

440443
**License:** Apache 2.0
441-
442-
---
443-
444-
Let me know if you want this written to your `README.md` or if you want to further tailor any section!

‎cmd/root.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,16 @@ func init() {
8484
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage")
8585
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for orchestrator storage")
8686
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxQueryTime", 60, "Clickhouse max query time for orchestrator storage")
87+
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-host", "", "PostgreSQL host for orchestrator storage")
88+
rootCmd.PersistentFlags().Int("storage-orchestrator-postgres-port", 5432, "PostgreSQL port for orchestrator storage")
89+
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-username", "", "PostgreSQL username for orchestrator storage")
90+
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-password", "", "PostgreSQL password for orchestrator storage")
91+
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-database", "", "PostgreSQL database for orchestrator storage")
92+
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-sslMode", "disable", "PostgreSQL SSL mode for orchestrator storage (disable, require, verify-ca, verify-full)")
93+
rootCmd.PersistentFlags().Int("storage-orchestrator-postgres-maxOpenConns", 50, "PostgreSQL max open connections for orchestrator storage")
94+
rootCmd.PersistentFlags().Int("storage-orchestrator-postgres-maxIdleConns", 25, "PostgreSQL max idle connections for orchestrator storage")
95+
rootCmd.PersistentFlags().Int("storage-orchestrator-postgres-maxConnLifetime", 300, "PostgreSQL max connection lifetime in seconds for orchestrator storage")
96+
rootCmd.PersistentFlags().Int("storage-orchestrator-postgres-connectTimeout", 10, "PostgreSQL connection timeout in seconds for orchestrator storage")
8797
rootCmd.PersistentFlags().String("storage-staging-clickhouse-host", "", "Clickhouse host for staging storage")
8898
rootCmd.PersistentFlags().String("storage-main-clickhouse-host", "", "Clickhouse host for main storage")
8999
rootCmd.PersistentFlags().String("storage-main-clickhouse-username", "", "Clickhouse username for main storage")
@@ -104,6 +114,16 @@ func init() {
104114
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage")
105115
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for staging storage")
106116
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxQueryTime", 60, "Clickhouse max query time for staging storage")
117+
rootCmd.PersistentFlags().String("storage-staging-postgres-host", "", "PostgreSQL host for staging storage")
118+
rootCmd.PersistentFlags().Int("storage-staging-postgres-port", 5432, "PostgreSQL port for staging storage")
119+
rootCmd.PersistentFlags().String("storage-staging-postgres-username", "", "PostgreSQL username for staging storage")
120+
rootCmd.PersistentFlags().String("storage-staging-postgres-password", "", "PostgreSQL password for staging storage")
121+
rootCmd.PersistentFlags().String("storage-staging-postgres-database", "", "PostgreSQL database for staging storage")
122+
rootCmd.PersistentFlags().String("storage-staging-postgres-sslMode", "disable", "PostgreSQL SSL mode for staging storage (disable, require, verify-ca, verify-full)")
123+
rootCmd.PersistentFlags().Int("storage-staging-postgres-maxOpenConns", 50, "PostgreSQL max open connections for staging storage")
124+
rootCmd.PersistentFlags().Int("storage-staging-postgres-maxIdleConns", 25, "PostgreSQL max idle connections for staging storage")
125+
rootCmd.PersistentFlags().Int("storage-staging-postgres-maxConnLifetime", 300, "PostgreSQL max connection lifetime in seconds for staging storage")
126+
rootCmd.PersistentFlags().Int("storage-staging-postgres-connectTimeout", 10, "PostgreSQL connection timeout in seconds for staging storage")
107127
rootCmd.PersistentFlags().String("api-host", "localhost:3000", "API host")
108128
rootCmd.PersistentFlags().String("api-basicAuth-username", "", "API basic auth username")
109129
rootCmd.PersistentFlags().String("api-basicAuth-password", "", "API basic auth password")
@@ -199,6 +219,26 @@ func init() {
199219
viper.BindPFlag("storage.orchestrator.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-disableTLS"))
200220
viper.BindPFlag("storage.orchestrator.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-enableParallelViewProcessing"))
201221
viper.BindPFlag("storage.orchestrator.clickhouse.maxQueryTime", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxQueryTime"))
222+
viper.BindPFlag("storage.orchestrator.postgres.host", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-host"))
223+
viper.BindPFlag("storage.orchestrator.postgres.port", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-port"))
224+
viper.BindPFlag("storage.orchestrator.postgres.username", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-username"))
225+
viper.BindPFlag("storage.orchestrator.postgres.password", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-password"))
226+
viper.BindPFlag("storage.orchestrator.postgres.database", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-database"))
227+
viper.BindPFlag("storage.orchestrator.postgres.sslMode", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-sslMode"))
228+
viper.BindPFlag("storage.orchestrator.postgres.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-maxOpenConns"))
229+
viper.BindPFlag("storage.orchestrator.postgres.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-maxIdleConns"))
230+
viper.BindPFlag("storage.orchestrator.postgres.maxConnLifetime", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-maxConnLifetime"))
231+
viper.BindPFlag("storage.orchestrator.postgres.connectTimeout", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-connectTimeout"))
232+
viper.BindPFlag("storage.staging.postgres.host", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-host"))
233+
viper.BindPFlag("storage.staging.postgres.port", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-port"))
234+
viper.BindPFlag("storage.staging.postgres.username", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-username"))
235+
viper.BindPFlag("storage.staging.postgres.password", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-password"))
236+
viper.BindPFlag("storage.staging.postgres.database", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-database"))
237+
viper.BindPFlag("storage.staging.postgres.sslMode", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-sslMode"))
238+
viper.BindPFlag("storage.staging.postgres.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-maxOpenConns"))
239+
viper.BindPFlag("storage.staging.postgres.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-maxIdleConns"))
240+
viper.BindPFlag("storage.staging.postgres.maxConnLifetime", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-maxConnLifetime"))
241+
viper.BindPFlag("storage.staging.postgres.connectTimeout", rootCmd.PersistentFlags().Lookup("storage-staging-postgres-connectTimeout"))
202242
viper.BindPFlag("api.host", rootCmd.PersistentFlags().Lookup("api-host"))
203243
viper.BindPFlag("api.basicAuth.username", rootCmd.PersistentFlags().Lookup("api-basicAuth-username"))
204244
viper.BindPFlag("api.basicAuth.password", rootCmd.PersistentFlags().Lookup("api-basicAuth-password"))

‎configs/config.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const (
6161

6262
type StorageConnectionConfig struct {
6363
Clickhouse *ClickhouseConfig `mapstructure:"clickhouse"`
64+
Postgres *PostgresConfig `mapstructure:"postgres"`
6465
}
6566

6667
type TableConfig struct {
@@ -86,6 +87,19 @@ type ClickhouseConfig struct {
8687
MaxQueryTime int `mapstructure:"maxQueryTime"`
8788
}
8889

90+
type PostgresConfig struct {
91+
Host string `mapstructure:"host"`
92+
Port int `mapstructure:"port"`
93+
Username string `mapstructure:"username"`
94+
Password string `mapstructure:"password"`
95+
Database string `mapstructure:"database"`
96+
SSLMode string `mapstructure:"sslMode"`
97+
MaxOpenConns int `mapstructure:"maxOpenConns"`
98+
MaxIdleConns int `mapstructure:"maxIdleConns"`
99+
MaxConnLifetime int `mapstructure:"maxConnLifetime"`
100+
ConnectTimeout int `mapstructure:"connectTimeout"`
101+
}
102+
89103
type RPCBatchRequestConfig struct {
90104
BlocksPerRequest int `mapstructure:"blocksPerRequest"`
91105
BatchDelay int `mapstructure:"batchDelay"`

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
github.com/gin-gonic/gin v1.10.0
99
github.com/gorilla/schema v1.4.1
1010
github.com/holiman/uint256 v1.3.2
11+
github.com/lib/pq v1.10.9
1112
github.com/prometheus/client_golang v1.20.4
1213
github.com/rs/zerolog v1.33.0
1314
github.com/spf13/cobra v1.8.1

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzW
175175
github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c=
176176
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
177177
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
178+
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
179+
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
178180
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
179181
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
180182
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=

‎indexer‎

-50.3 MB
Binary file not shown.

‎internal/storage/connector.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,9 @@ func NewStorageConnector(cfg *config.StorageConfig) (IStorage, error) {
143143
func NewConnector[T any](cfg *config.StorageConnectionConfig) (T, error) {
144144
var conn interface{}
145145
var err error
146-
if cfg.Clickhouse != nil {
146+
if cfg.Postgres != nil {
147+
conn, err = NewPostgresConnector(cfg.Postgres)
148+
} else if cfg.Clickhouse != nil {
147149
conn, err = NewClickHouseConnector(cfg.Clickhouse)
148150
} else {
149151
return *new(T), fmt.Errorf("no storage driver configured")

0 commit comments

Comments
 (0)