build steps in readme + test_config

Author: nischitpra

README.md

@@ -33,6 +33,33 @@ go build -o main -tags=production
 # Default: http://localhost:3000
 ```
 
+### 🏃‍♂️ Quick Start with Hybrid Setup (PostgreSQL + ClickHouse)
+
+For testing the staging flow with PostgreSQL for orchestration/staging and ClickHouse for main storage:
+
+```bash
+# 1. Build the application
+go build -o insight .
+
+# 2. Start databases (PostgreSQL + ClickHouse)
+docker compose up -d
+
+# 3. Run the orchestrator with test configuration
+./insight orchestrator --config configs/test_config.yml
+
+# 4. (Optional) Start the API server
+./insight api --config configs/test_config.yml
+
+# 5. Monitor the databases
+# PostgreSQL (staging): docker exec insight-postgres-1 psql -U admin -d insight
+# ClickHouse (main): clickhouse-client --host localhost --port 9440 --user admin --password password
+```
+
+**Database Connections:**
+- **PostgreSQL**: `localhost:5432` (staging & orchestrator storage)
+- **ClickHouse**: `localhost:9440` (main storage)
+- **Credentials**: `admin` / `password`
+
 ---
 
 ## 🏗 How It Works

configs/test_config.yml

@@ -0,0 +1,77 @@
+rpc:
+  url: https://1.rpc.thirdweb.com
+  chainId: "1"
+
+log:
+  level: debug
+  prettify: true
+
+poller:
+  enabled: true
+  interval: 2000
+  blocksPerPoll: 100
+
+committer:
+  enabled: true
+  interval: 2000
+  blocksPerCommit: 100
+
+storage:
+  main:
+    clickhouse:
+      host: localhost
+      port: 9440
+      username: admin
+      password: password
+      database: default
+      disableTLS: true
+      asyncInsert: true
+      maxRowsPerInsert: 1000
+      maxOpenConns: 50
+      maxIdleConns: 10
+  
+  staging:
+    postgres:
+      host: localhost
+      port: 5432
+      username: admin
+      password: password
+      database: insight
+      sslMode: disable
+      maxOpenConns: 50
+      maxIdleConns: 10
+      maxConnLifetime: 300
+      connectTimeout: 10
+  
+  orchestrator:
+    postgres:
+      host: localhost
+      port: 5432
+      username: admin
+      password: password
+      database: insight
+      sslMode: disable
+      maxOpenConns: 50
+      maxIdleConns: 10
+      maxConnLifetime: 300
+      connectTimeout: 10
+
+api:
+  host: localhost:3000
+  basicAuth:
+    username: admin
+    password: admin
+
+publisher:
+  enabled: false
+
+validation:
+  mode: minimal 
+
+# Work mode configuration - Controls system behavior based on blockchain state
+workMode:
+  # Interval in minutes to check if system should switch between live/historical mode
+  checkIntervalMinutes: 10
+  # Block number threshold to determine if system is in "live mode" (near chain head)
+  # Setting this very high forces backfill mode for testing
+  liveModeThreshold: 1000000 

docker-compose.yml

@@ -9,6 +9,18 @@ services:
     profiles:
       - redis
 
+  # PostgreSQL database
+  postgres:
+    image: postgres:15
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_DB: insight
+      POSTGRES_USER: admin
+      POSTGRES_PASSWORD: password
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
   # http://localhost:8123/play <- web ui
   clickhouse:
     image: clickhouse/clickhouse-server:latest
@@ -81,6 +93,7 @@ services:
 
 volumes:
   redis_data:
+  postgres_data:
   clickhouse_data:
   grafana_data: