Skip to content

Commit 609f752

Browse files
nischitpranischitpra
committed
readonly clickhouse
1 parent 13db8af commit 609f752

File tree

8 files changed

+786
-20
lines changed

8 files changed

+786
-20
lines changed

‎configs/config.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"fmt"
66
"os"
7+
"strconv"
78
"strings"
89

910
"github.com/rs/zerolog/log"
@@ -85,6 +86,13 @@ type ClickhouseConfig struct {
8586
ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"`
8687
EnableParallelViewProcessing bool `mapstructure:"enableParallelViewProcessing"`
8788
MaxQueryTime int `mapstructure:"maxQueryTime"`
89+
90+
// Readonly configuration for API endpoints
91+
ReadonlyHost string `mapstructure:"readonlyHost"`
92+
ReadonlyPort int `mapstructure:"readonlyPort"`
93+
ReadonlyUsername string `mapstructure:"readonlyUsername"`
94+
ReadonlyPassword string `mapstructure:"readonlyPassword"`
95+
ReadonlyDatabase string `mapstructure:"readonlyDatabase"`
8896
}
8997

9098
type PostgresConfig struct {
@@ -282,5 +290,35 @@ func setCustomJSONConfigs() error {
282290
Cfg.Storage.Main.Clickhouse.ChainBasedConfig = orchestratorChainConfig
283291
}
284292
}
293+
294+
// Load readonly ClickHouse configuration from environment variables
295+
if readonlyHost := os.Getenv("CLICKHOUSE_HOST_READONLY"); readonlyHost != "" {
296+
if Cfg.Storage.Main.Clickhouse != nil {
297+
Cfg.Storage.Main.Clickhouse.ReadonlyHost = readonlyHost
298+
}
299+
}
300+
if readonlyPort := os.Getenv("CLICKHOUSE_PORT_READONLY"); readonlyPort != "" {
301+
if port, err := strconv.Atoi(readonlyPort); err == nil {
302+
if Cfg.Storage.Main.Clickhouse != nil {
303+
Cfg.Storage.Main.Clickhouse.ReadonlyPort = port
304+
}
305+
}
306+
}
307+
if readonlyUsername := os.Getenv("CLICKHOUSE_USER_READONLY"); readonlyUsername != "" {
308+
if Cfg.Storage.Main.Clickhouse != nil {
309+
Cfg.Storage.Main.Clickhouse.ReadonlyUsername = readonlyUsername
310+
}
311+
}
312+
if readonlyPassword := os.Getenv("CLICKHOUSE_PASSWORD_READONLY"); readonlyPassword != "" {
313+
if Cfg.Storage.Main.Clickhouse != nil {
314+
Cfg.Storage.Main.Clickhouse.ReadonlyPassword = readonlyPassword
315+
}
316+
}
317+
if readonlyDatabase := os.Getenv("CLICKHOUSE_DATABASE_READONLY"); readonlyDatabase != "" {
318+
if Cfg.Storage.Main.Clickhouse != nil {
319+
Cfg.Storage.Main.Clickhouse.ReadonlyDatabase = readonlyDatabase
320+
}
321+
}
322+
285323
return nil
286324
}

‎configs/config_readonly_example.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Example configuration for readonly ClickHouse API endpoints
2+
# This configuration allows you to use a separate readonly ClickHouse instance
3+
# for user-facing API queries while keeping the main orchestration flow unchanged
4+
5+
storage:
6+
main:
7+
clickhouse:
8+
# Main ClickHouse configuration (for orchestration)
9+
host: "localhost"
10+
port: 9000
11+
username: "default"
12+
password: "password"
13+
database: "insight"
14+
15+
# Readonly ClickHouse configuration (for API endpoints)
16+
# These environment variables will override the main config for API queries
17+
readonlyHost: "${CLICKHOUSE_HOST_READONLY}" # e.g., "readonly-clickhouse.example.com"
18+
readonlyPort: "${CLICKHOUSE_PORT_READONLY}" # e.g., 9000
19+
readonlyUsername: "${CLICKHOUSE_USER_READONLY}" # e.g., "readonly_user"
20+
readonlyPassword: "${CLICKHOUSE_PASSWORD_READONLY}" # e.g., "readonly_password"
21+
readonlyDatabase: "${CLICKHOUSE_DATABASE_READONLY}" # e.g., "insight_readonly"
22+
23+
# Other ClickHouse settings
24+
disableTLS: false
25+
maxOpenConns: 100
26+
maxIdleConns: 10
27+
maxRowsPerInsert: 100000
28+
enableParallelViewProcessing: true
29+
maxQueryTime: 300
30+
31+
# Environment variables to set:
32+
# CLICKHOUSE_HOST_READONLY=readonly-clickhouse.example.com
33+
# CLICKHOUSE_PORT_READONLY=9000
34+
# CLICKHOUSE_USER_READONLY=readonly_user
35+
# CLICKHOUSE_PASSWORD_READONLY=readonly_password
36+
# CLICKHOUSE_DATABASE_READONLY=insight_readonly
37+
38+
# How it works:
39+
# 1. When readonly environment variables are set, API endpoints will use the readonly ClickHouse instance
40+
# 2. The orchestration flow (indexer, committer, etc.) will continue to use the main ClickHouse instance
41+
# 3. This provides read/write separation and allows you to scale your readonly queries independently
42+
# 4. If readonly configuration is not provided, API endpoints will fall back to the main ClickHouse instance

‎docs/README_READONLY_CLICKHOUSE.md‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
# Readonly ClickHouse for API Endpoints
2+
3+
This document explains how to configure and use a readonly ClickHouse instance for user-facing API endpoints while keeping the main orchestration flow unchanged.
4+
5+
## Overview
6+
7+
The readonly ClickHouse feature allows you to:
8+
- Use a separate, read-only ClickHouse instance for API queries
9+
- Keep the main ClickHouse instance for orchestration (indexing, committing, etc.)
10+
- Scale read and write operations independently
11+
- Improve API performance by using dedicated read replicas
12+
13+
## Configuration
14+
15+
### Environment Variables
16+
17+
Set the following environment variables to enable readonly ClickHouse:
18+
19+
```bash
20+
export CLICKHOUSE_HOST_READONLY="readonly-clickhouse.example.com"
21+
export CLICKHOUSE_PORT_READONLY=9000
22+
export CLICKHOUSE_USER_READONLY="readonly_user"
23+
export CLICKHOUSE_PASSWORD_READONLY="readonly_password"
24+
export CLICKHOUSE_DATABASE_READONLY="insight_readonly"
25+
```
26+
27+
### Configuration File
28+
29+
You can also set these values in your configuration file:
30+
31+
```yaml
32+
storage:
33+
main:
34+
clickhouse:
35+
# Main ClickHouse configuration (for orchestration)
36+
host: "localhost"
37+
port: 9000
38+
username: "default"
39+
password: "password"
40+
database: "insight"
41+
42+
# Readonly ClickHouse configuration (for API endpoints)
43+
readonlyHost: "readonly-clickhouse.example.com"
44+
readonlyPort: 9000
45+
readonlyUsername: "readonly_user"
46+
readonlyPassword: "readonly_password"
47+
readonlyDatabase: "insight_readonly"
48+
```
49+
50+
## How It Works
51+
52+
### Automatic Detection
53+
54+
The system automatically detects if readonly configuration is available:
55+
56+
1. **With readonly config**: API endpoints use the readonly ClickHouse instance
57+
2. **Without readonly config**: API endpoints fall back to the main ClickHouse instance
58+
59+
### Affected Endpoints
60+
61+
The following API endpoints will use the readonly ClickHouse instance when configured:
62+
63+
- `GET /{chainId}/blocks` - Block queries
64+
- `GET /{chainId}/transactions` - Transaction queries
65+
- `GET /{chainId}/events` - Event/log queries
66+
- `GET /{chainId}/transfers` - Token transfer queries
67+
- `GET /{chainId}/balances/{owner}` - Token balance queries
68+
- `GET /{chainId}/holders/{address}` - Token holder queries
69+
- `GET /{chainId}/search/{input}` - Search queries
70+
71+
### Unaffected Operations
72+
73+
The following operations continue to use the main ClickHouse instance:
74+
75+
- Block indexing and polling
76+
- Transaction processing
77+
- Event/log processing
78+
- Staging data operations
79+
- Orchestration flow (committer, failure recovery, etc.)
80+
81+
## Implementation Details
82+
83+
### Readonly Connector
84+
85+
The `ClickHouseReadonlyConnector` implements the same interface as the main connector but:
86+
87+
- Only allows read operations
88+
- Panics on write operations (ensuring readonly behavior)
89+
- Uses readonly connection parameters
90+
- Falls back to main config if readonly config is incomplete
91+
92+
### Storage Factory
93+
94+
The `NewReadonlyConnector` function creates readonly connectors:
95+
96+
```go
97+
// For API endpoints (readonly if configured)
98+
storage.NewReadonlyConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
99+
100+
// For orchestration (always main connector)
101+
storage.NewConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
102+
```
103+
104+
## Setup Instructions
105+
106+
### 1. Create Readonly ClickHouse Instance
107+
108+
Set up a ClickHouse replica or read-only instance:
109+
110+
```sql
111+
-- On your readonly ClickHouse instance
112+
CREATE DATABASE insight_readonly;
113+
-- Grant readonly permissions to readonly_user
114+
GRANT SELECT ON insight_readonly.* TO readonly_user;
115+
```
116+
117+
### 2. Set Environment Variables
118+
119+
```bash
120+
export CLICKHOUSE_HOST_READONLY="your-readonly-host"
121+
export CLICKHOUSE_PORT_READONLY=9000
122+
export CLICKHOUSE_USER_READONLY="readonly_user"
123+
export CLICKHOUSE_PASSWORD_READONLY="readonly_password"
124+
export CLICKHOUSE_DATABASE_READONLY="insight_readonly"
125+
```
126+
127+
### 3. Restart API Server
128+
129+
Restart your API server to pick up the new configuration:
130+
131+
```bash
132+
./insight api
133+
```
134+
135+
### 4. Verify Configuration
136+
137+
Check the logs to confirm which connector is being used:
138+
139+
```
140+
INFO Using readonly ClickHouse connector for API endpoints
141+
```
142+
143+
## Monitoring and Troubleshooting
144+
145+
### Log Messages
146+
147+
- **"Using readonly ClickHouse connector for API endpoints"** - Readonly mode active
148+
- **"Using regular ClickHouse connector for API endpoints"** - Fallback to main connector
149+
150+
### Common Issues
151+
152+
1. **Connection refused**: Check readonly ClickHouse host/port
153+
2. **Authentication failed**: Verify readonly username/password
154+
3. **Database not found**: Ensure readonly database exists
155+
4. **Permission denied**: Grant SELECT permissions to readonly user
156+
157+
### Testing
158+
159+
Test the readonly connection:
160+
161+
```bash
162+
# Test readonly connection
163+
clickhouse-client --host=readonly-host --port=9000 --user=readonly_user --password=readonly_password --database=insight_readonly -q "SELECT 1"
164+
```
165+
166+
## Performance Considerations
167+
168+
### Read Replicas
169+
170+
- Use ClickHouse read replicas for better performance
171+
- Consider geographic distribution for global users
172+
- Monitor replica lag to ensure data consistency
173+
174+
### Connection Pooling
175+
176+
The readonly connector uses the same connection pool settings as the main connector:
177+
178+
```yaml
179+
maxOpenConns: 100
180+
maxIdleConns: 10
181+
```
182+
183+
### Query Optimization
184+
185+
- Readonly instances can be optimized for query performance
186+
- Consider different ClickHouse settings for readonly vs. write instances
187+
- Use materialized views on readonly instances for complex queries
188+
189+
## Security
190+
191+
### Network Security
192+
193+
- Restrict readonly ClickHouse to internal networks
194+
- Use VPN or private subnets for readonly access
195+
- Consider ClickHouse's built-in network security features
196+
197+
### User Permissions
198+
199+
- Create dedicated readonly user with minimal permissions
200+
- Grant only SELECT permissions on required tables
201+
- Regularly rotate readonly user passwords
202+
203+
### Data Access
204+
205+
- Readonly users cannot modify data
206+
- No risk of accidental data corruption
207+
- Audit logs show readonly access patterns
208+
209+
## Migration
210+
211+
### From Single Instance
212+
213+
1. Set up readonly ClickHouse instance
214+
2. Configure environment variables
215+
3. Restart API server
216+
4. Monitor performance and errors
217+
5. Gradually migrate more endpoints if needed
218+
219+
### Rollback
220+
221+
To rollback to main ClickHouse:
222+
223+
1. Remove readonly environment variables
224+
2. Restart API server
225+
3. API endpoints will automatically use main connector
226+
227+
## Support
228+
229+
For issues or questions about the readonly ClickHouse feature:
230+
231+
1. Check the logs for error messages
232+
2. Verify ClickHouse connectivity
233+
3. Review configuration parameters
234+
4. Check ClickHouse server logs
235+
5. Contact the development team

‎insight‎

100755100644
-49.2 MB
Binary file not shown.

‎internal/handlers/logs_handlers.go‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package handlers
22

33
import (
44
"net/http"
5-
"sync"
65

76
"github.com/ethereum/go-ethereum/accounts/abi"
87
"github.com/gin-gonic/gin"
@@ -13,13 +12,6 @@ import (
1312
"github.com/thirdweb-dev/indexer/internal/storage"
1413
)
1514

16-
// package-level variables
17-
var (
18-
mainStorage storage.IMainStorage
19-
storageOnce sync.Once
20-
storageErr error
21-
)
22-
2315
// @Summary Get all logs
2416
// @Description Retrieve all logs across all contracts
2517
// @Tags events
@@ -221,18 +213,6 @@ func decodeLogsIfNeeded(chainId string, logs []common.Log, eventABI *abi.Event,
221213
return nil
222214
}
223215

224-
func getMainStorage() (storage.IMainStorage, error) {
225-
storageOnce.Do(func() {
226-
var err error
227-
mainStorage, err = storage.NewConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
228-
if err != nil {
229-
storageErr = err
230-
log.Error().Err(err).Msg("Error creating storage connector")
231-
}
232-
})
233-
return mainStorage, storageErr
234-
}
235-
236216
func sendJSONResponse(c *gin.Context, response interface{}) {
237217
c.JSON(http.StatusOK, response)
238218
}

0 commit comments

Comments
 (0)