Skip to content

Commit da510c3

Browse files
placer14placer14
authored
feat(docker): Update docker-compose to use lily worker pattern (#1044)
* feat(docker): Update docker-compose to use lily worker pattern * fix: Snapshot envvar; prom host; env quotes * chore: Address comments Co-authored-by: Mike Greenberg <[email protected]>
1 parent 593dad6 commit da510c3

File tree

9 files changed

+162
-36
lines changed

9 files changed

+162
-36
lines changed

‎build/grafana/provisioning/datasources/datasource.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ datasources:
1818
# <int> org id. will default to orgId 1 if not specified
1919
orgId: 1
2020
# <string> url
21-
url: http://localhost:9090
21+
url: http://prometheus:9090
2222
# <string> database password, if used
2323
password:
2424
# <string> database user, if used

‎build/lily/docker_init.sh‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,17 @@ set -exo pipefail
44

55
mkdir -p ${LILY_REPO}/keystore
66

7-
if [[ ! -z "${_LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT}" ]]; then
8-
# import snapshot when _LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT is set
7+
8+
if [[ ! -z "${LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT}" ]]; then
9+
# set default snapshot path if not already defined
10+
snapshot="${LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH:-https://snapshots.mainnet.filops.net/minimal/latest}"
11+
12+
# import snapshot when LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT is set
913
if [[ -f "${LILY_REPO}/datastore/_imported" ]]; then
1014
echo "Skipping import, found ${LILY_REPO}/datastore/_imported file."
1115
else
12-
echo "Importing snapshot from https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car..."
13-
lily init --import-snapshot="https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car"
14-
16+
echo "Importing snapshot from ${snapshot}"
17+
lily init --import-snapshot=${snapshot}
1518
status=$?
1619
if [ $status -eq 0 ]; then
1720
touch "/var/lib/lily/datastore/_imported"
@@ -22,6 +25,6 @@ else
2225
lily init
2326
fi
2427

25-
chmod 0600 ${LILY_REPO}/keystore/*
28+
chmod -R 0600 ${LILY_REPO}/keystore
2629

2730
lily $@

‎build/lily/docker.env‎ renamed to ‎build/lily/notifier.env‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,20 @@ LILY_REPO=/var/lib/lily
55
#LILY_CONFIG=/var/lib/lily/config.toml
66

77
# Postgres URL may be overridden
8-
#LILY_STORAGE_POSTGRESQL_DB_URL=postgres://user:pass@host:port/database?options
8+
LILY_STORAGE_POSTGRESQL_DB_URL=postgres://postgres:password@timescaledb:5432/postgres?sslmode=disable
99

1010
# Enable IMPORT_SNAPSHOT below to use snapshot on lily startup
11-
#_LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true
11+
#LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true
12+
#LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH=
1213

1314
# Debugging options
1415
#LILY_TRACING=true
15-
LILY_PROMETHEUS_PORT="prometheus:9090"
16+
LILY_PROMETHEUS_PORT=0.0.0.0:9090
1617

1718
# Logging options
1819
GOLOG_LOG_FMT=json
1920
GOLOG_LOG_LEVEL=info
20-
LILY_LOG_LEVEL_NAMED="vm:error,badgerbs:error"
21+
LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error
22+
23+
# Queue configuration
24+
LILY_REDIS_ADDR=redis-queue:6379

‎build/lily/notifier_config.toml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[API]
2+
ListenAddress = "/ip4/0.0.0.0/tcp/1234/http"
3+
[Libp2p]
4+
ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"]
5+
ConnMgrLow = 400
6+
ConnMgrHigh = 500
7+
ConnMgrGrace = "5m0s"
8+
[Queue]
9+
[Queue.Notifiers]
10+
[Queue.Notifiers.Notifier1]
11+
Network = "tcp"
12+
Addr = "redis-queue:6379"
13+
DB = 0
14+
PoolSize = 0
15+
[Storage]
16+
[Storage.Postgresql]
17+
[Storage.Postgresql.postgres]
18+
SchemaName = "lily"
19+
URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL"
20+
ApplicationName = "lily"
21+
PoolSize = 20
22+
AllowUpsert = false

‎build/lily/redis-queue.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# FOR DEV ONLY: disabled authentication
2+
ALLOW_EMPTY_PASSWORD=yes

‎build/lily/worker.env‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Path for lily state should be on a persisted volume
2+
LILY_REPO=/var/lib/lily
3+
4+
# Config path default is below but may be customized
5+
#LILY_CONFIG=/var/lib/lily/config.toml
6+
7+
# Postgres URL may be overridden
8+
LILY_STORAGE_POSTGRESQL_DB_URL=postgres://postgres:password@timescaledb:5432/postgres?sslmode=disable
9+
10+
# Enable IMPORT_SNAPSHOT below to use snapshot on lily startup
11+
#LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true
12+
#LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH=
13+
14+
# Debugging options
15+
#LILY_TRACING=true
16+
LILY_PROMETHEUS_PORT=0.0.0.0:9091
17+
18+
# Logging options
19+
GOLOG_LOG_FMT=json
20+
GOLOG_LOG_LEVEL=info
21+
LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error
22+
23+
# Queue configuration
24+
LILY_REDIS_ADDR=redis-queue:6379

‎build/lily/worker_config.toml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[API]
2+
ListenAddress = "/ip4/0.0.0.0/tcp/1234/http"
3+
[Libp2p]
4+
ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"]
5+
ConnMgrLow = 400
6+
ConnMgrHigh = 500
7+
ConnMgrGrace = "5m0s"
8+
[Queue]
9+
[Queue.Workers]
10+
[Queue.Workers.Worker1]
11+
[Queue.Workers.Worker1.RedisConfig]
12+
Network = "tcp"
13+
Addr = "redis-queue:6379"
14+
DB = 0
15+
PoolSize = 0
16+
[Queue.Workers.Worker1.WorkerConfig]
17+
Concurrency = 1
18+
LoggerLevel = "debug"
19+
WatchQueuePriority = 5
20+
FillQueuePriority = 3
21+
IndexQueuePriority = 1
22+
WalkQueuePriority = 1
23+
StrictPriority = false
24+
ShutdownTimeout = 30000000000
25+
[Storage]
26+
[Storage.Postgresql]
27+
[Storage.Postgresql.postgres]
28+
SchemaName = "lily"
29+
URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL"
30+
ApplicationName = "lily"
31+
PoolSize = 20
32+
AllowUpsert = false

‎build/prometheus/prometheus.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ scrape_configs:
44
scrape_interval: 5s
55

66
static_configs:
7-
- targets: ['localhost:9991']
7+
- targets: ['notifier:9090', 'worker:9091']

‎docker-compose.yml‎

Lines changed: 63 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ services:
3737
- '--web.console.templates=/usr/share/prometheus/consoles'
3838
ports:
3939
- 9090:9090
40-
network_mode: "host"
40+
#network_mode: "host"
4141
restart: always
4242

4343
grafana:
@@ -47,38 +47,77 @@ services:
4747
- prometheus
4848
ports:
4949
- 3000:3000
50-
network_mode: "host"
50+
#network_mode: "host"
5151
volumes:
5252
- grafana_data:/var/lib/grafana
5353
- ./build/grafana/provisioning/:/etc/grafana/provisioning/
5454
env_file:
5555
- ./build/grafana/config.monitoring
5656
restart: always
5757

58-
#lily:
59-
#container_name: lily
60-
## Select (only one) image
61-
#image: filecoin/lily:v0.8.0
62-
#image: filecoin/lily:v0.8.0-calibnet
63-
#image: filecoin/lily:v0.8.0-calibnet-dev
64-
#env_file:
65-
## Check envvars for configurable options
66-
#- ./build/lily/docker.env
67-
#depends_on:
68-
#- prometheus
69-
#- timescaledb
70-
#- jaeger
71-
#ports:
72-
#- 1234:1234
73-
#volumes:
74-
#- lily_data:/var/lib/lily
75-
#- ./build/lily/docker_init.sh:/usr/bin/docker_init.sh
76-
#entrypoint: /usr/bin/docker_init.sh
77-
#command:
78-
#- daemon
58+
redis-queue:
59+
image: bitnami/redis:7.0
60+
env_file:
61+
# Check envvars for configurable options
62+
- ./build/lily/redis-queue.env
63+
ports:
64+
- 6379:6379
65+
restart: always
66+
67+
notifier:
68+
image: filecoin/lily:v0.11.0
69+
env_file:
70+
# Check envvars for configurable options
71+
- ./build/lily/notifier.env
72+
depends_on:
73+
- prometheus
74+
- timescaledb
75+
- jaeger
76+
- redis-queue
77+
ports:
78+
- 1234:1234
79+
volumes:
80+
# holds lily datastore repo
81+
- lily_notifier_data:/var/lib/lily
82+
# persist params through restarts
83+
- lily_notifier_tmp:/var/tmp/filecoin-proof-parameters
84+
# notifier-specific config
85+
- ./build/lily/notifier_config.toml:/var/lib/lily/config.toml
86+
- ./build/lily/docker_init.sh:/usr/bin/docker_init.sh
87+
entrypoint: /usr/bin/docker_init.sh
88+
command:
89+
- "daemon --bootstrap=false"
90+
91+
worker:
92+
image: filecoin/lily:v0.11.0
93+
env_file:
94+
# Check envvars for configurable options
95+
- ./build/lily/worker.env
96+
depends_on:
97+
- prometheus
98+
- timescaledb
99+
- jaeger
100+
- redis-queue
101+
- notifier
102+
ports:
103+
- 1234
104+
volumes:
105+
# holds lily datastore repo
106+
- lily_worker_data:/var/lib/lily
107+
# persist params through restarts
108+
- lily_worker_tmp:/var/tmp/filecoin-proof-parameters
109+
# notifier-specific config
110+
- ./build/lily/worker_config.toml:/var/lib/lily/config.toml
111+
- ./build/lily/docker_init.sh:/usr/bin/docker_init.sh
112+
entrypoint: /usr/bin/docker_init.sh
113+
command:
114+
- "daemon --bootstrap=false"
79115

80116
volumes:
81117
timescaledb: {}
82118
prometheus_data: {}
83119
grafana_data: {}
84-
lily_data: {}
120+
lily_notifier_data: {}
121+
lily_worker_data: {}
122+
lily_notifier_tmp: {}
123+
lily_worker_tmp: {}

0 commit comments

Comments
 (0)