Skip to content

Commit 6773190

Browse files
nikhilsinhaparseablenikhilsinhaparseable
authored
add docker-compose for distributed mode testing (#777)
This PR adds docker-compose for distributed workflow updated to test both standalone and distributed modes. fixes : #644
1 parent 5059d71 commit 6773190

File tree

2 files changed

+269
-1
lines changed

2 files changed

+269
-1
lines changed

‎.github/workflows/integration-test.yaml

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ on:
1111
jobs:
1212

1313
docker-compose-test:
14-
name: Quest Smoke and Load Tests
14+
name: Quest Smoke and Load Tests for Standalone deployments
1515
runs-on: ubuntu-latest
1616
steps:
1717
- name: Checkout
@@ -21,3 +21,15 @@ jobs:
2121
- name: Stop compose
2222
if: always()
2323
run: docker-compose -f docker-compose-test.yaml down
24+
25+
docker-compose-distributed-test:
26+
name: Quest Smoke and Load Tests for Distributed deployments
27+
runs-on: ubuntu-latest
28+
steps:
29+
- name: Checkout
30+
uses: actions/checkout@v3
31+
- name: Start compose
32+
run: docker-compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest
33+
- name: Stop compose
34+
if: always()
35+
run: docker-compose -f docker-compose-distributed-test.yaml down

‎docker-compose-distributed-test.yaml

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
version: "3.7"
2+
networks:
3+
parseable-internal:
4+
services:
5+
# minio
6+
minio:
7+
image: minio/minio:RELEASE.2023-02-10T18-48-39Z
8+
entrypoint:
9+
- sh
10+
- -euc
11+
- |
12+
mkdir -p /tmp/minio/parseable && \
13+
minio server /tmp/minio
14+
environment:
15+
- MINIO_ROOT_USER=parseable
16+
- MINIO_ROOT_PASSWORD=supersecret
17+
- MINIO_UPDATE=off
18+
ports:
19+
- 9000:9000
20+
healthcheck:
21+
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
22+
interval: 15s
23+
timeout: 20s
24+
retries: 5
25+
networks:
26+
- parseable-internal
27+
# query server
28+
parseable-query:
29+
build:
30+
context: .
31+
dockerfile: Dockerfile
32+
command: ["parseable", "s3-store"]
33+
ports:
34+
- 8000:8000
35+
environment:
36+
- P_S3_URL=http://minio:9000
37+
- P_S3_ACCESS_KEY=parseable
38+
- P_S3_SECRET_KEY=supersecret
39+
- P_S3_REGION=us-east-1
40+
- P_S3_BUCKET=parseable
41+
- P_STAGING_DIR=/tmp/data
42+
- P_USERNAME=parseableadmin
43+
- P_PASSWORD=parseableadmin
44+
- P_CHECK_UPDATE=false
45+
- P_PARQUET_COMPRESSION_ALGO=snappy
46+
- P_MODE=query
47+
networks:
48+
- parseable-internal
49+
healthcheck:
50+
test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
51+
interval: 15s
52+
timeout: 20s
53+
retries: 5
54+
depends_on:
55+
- minio
56+
deploy:
57+
restart_policy:
58+
condition: on-failure
59+
delay: 20s
60+
max_attempts: 3
61+
# ingest server one
62+
parseable-ingest-one:
63+
build:
64+
context: .
65+
dockerfile: Dockerfile
66+
command: ["parseable", "s3-store"]
67+
ports:
68+
- 8000
69+
environment:
70+
- P_S3_URL=http://minio:9000
71+
- P_S3_ACCESS_KEY=parseable
72+
- P_S3_SECRET_KEY=supersecret
73+
- P_S3_REGION=us-east-1
74+
- P_S3_BUCKET=parseable
75+
- P_STAGING_DIR=/tmp/data
76+
- P_USERNAME=parseableadmin
77+
- P_PASSWORD=parseableadmin
78+
- P_CHECK_UPDATE=false
79+
- P_PARQUET_COMPRESSION_ALGO=snappy
80+
- P_MODE=ingest
81+
- P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
82+
networks:
83+
- parseable-internal
84+
healthcheck:
85+
test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
86+
interval: 15s
87+
timeout: 20s
88+
retries: 5
89+
depends_on:
90+
- parseable-query
91+
- minio
92+
deploy:
93+
restart_policy:
94+
condition: on-failure
95+
delay: 20s
96+
max_attempts: 3
97+
# ingest server two
98+
parseable-ingest-two:
99+
build:
100+
context: .
101+
dockerfile: Dockerfile
102+
command: ["parseable", "s3-store"]
103+
ports:
104+
- 8000
105+
environment:
106+
- P_S3_URL=http://minio:9000
107+
- P_S3_ACCESS_KEY=parseable
108+
- P_S3_SECRET_KEY=supersecret
109+
- P_S3_REGION=us-east-1
110+
- P_S3_BUCKET=parseable
111+
- P_STAGING_DIR=/tmp/data
112+
- P_USERNAME=parseableadmin
113+
- P_PASSWORD=parseableadmin
114+
- P_CHECK_UPDATE=false
115+
- P_PARQUET_COMPRESSION_ALGO=snappy
116+
- P_MODE=ingest
117+
- P_INGESTOR_ENDPOINT=parseable-ingest-two:8000
118+
networks:
119+
- parseable-internal
120+
healthcheck:
121+
test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
122+
interval: 15s
123+
timeout: 20s
124+
retries: 5
125+
depends_on:
126+
- parseable-query
127+
- minio
128+
deploy:
129+
restart_policy:
130+
condition: on-failure
131+
delay: 20s
132+
max_attempts: 3
133+
# ingest server three
134+
parseable-ingest-three:
135+
build:
136+
context: .
137+
dockerfile: Dockerfile
138+
command: ["parseable", "s3-store"]
139+
ports:
140+
- 8000
141+
environment:
142+
- P_S3_URL=http://minio:9000
143+
- P_S3_ACCESS_KEY=parseable
144+
- P_S3_SECRET_KEY=supersecret
145+
- P_S3_REGION=us-east-1
146+
- P_S3_BUCKET=parseable
147+
- P_STAGING_DIR=/tmp/data
148+
- P_USERNAME=parseableadmin
149+
- P_PASSWORD=parseableadmin
150+
- P_CHECK_UPDATE=false
151+
- P_PARQUET_COMPRESSION_ALGO=snappy
152+
- P_MODE=ingest
153+
- P_INGESTOR_ENDPOINT=parseable-ingest-three:8000
154+
networks:
155+
- parseable-internal
156+
healthcheck:
157+
test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
158+
interval: 15s
159+
timeout: 20s
160+
retries: 5
161+
depends_on:
162+
- parseable-query
163+
- minio
164+
deploy:
165+
restart_policy:
166+
condition: on-failure
167+
delay: 20s
168+
max_attempts: 3
169+
# ingest server four
170+
parseable-ingest-four:
171+
build:
172+
context: .
173+
dockerfile: Dockerfile
174+
command: ["parseable", "s3-store"]
175+
ports:
176+
- 8000
177+
environment:
178+
- P_S3_URL=http://minio:9000
179+
- P_S3_ACCESS_KEY=parseable
180+
- P_S3_SECRET_KEY=supersecret
181+
- P_S3_REGION=us-east-1
182+
- P_S3_BUCKET=parseable
183+
- P_STAGING_DIR=/tmp/data
184+
- P_USERNAME=parseableadmin
185+
- P_PASSWORD=parseableadmin
186+
- P_CHECK_UPDATE=false
187+
- P_PARQUET_COMPRESSION_ALGO=snappy
188+
- P_MODE=ingest
189+
- P_INGESTOR_ENDPOINT=parseable-ingest-four:8000
190+
networks:
191+
- parseable-internal
192+
healthcheck:
193+
test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
194+
interval: 15s
195+
timeout: 20s
196+
retries: 5
197+
depends_on:
198+
- parseable-query
199+
- minio
200+
deploy:
201+
restart_policy:
202+
condition: on-failure
203+
delay: 20s
204+
max_attempts: 3
205+
# penguin lb
206+
penguin:
207+
platform: linux/amd64
208+
image: parseable/penguin:edge
209+
command: ["./target/release/penguin"]
210+
environment:
211+
- P_ADDR_ONE=parseable-ingest-one:8000
212+
- P_ADDR_TWO=parseable-ingest-two:8000
213+
- P_ADDR_THREE=parseable-ingest-three:8000
214+
- P_ADDR_FOUR=parseable-ingest-four:8000
215+
ports:
216+
- 6188:6188
217+
depends_on:
218+
- parseable-query
219+
- minio
220+
- parseable-ingest-one
221+
- parseable-ingest-two
222+
- parseable-ingest-three
223+
- parseable-ingest-four
224+
networks:
225+
- parseable-internal
226+
quest:
227+
platform: linux/amd64
228+
image: ghcr.io/parseablehq/quest:main
229+
command:
230+
[
231+
"load",
232+
"http://parseable-query:8000",
233+
"parseableadmin",
234+
"parseableadmin",
235+
"20",
236+
"10",
237+
"5m",
238+
"minio:9000",
239+
"parseable",
240+
"supersecret",
241+
"parseable",
242+
"http://penguin:6188",
243+
"parseableadmin",
244+
"parseableadmin",
245+
]
246+
networks:
247+
- parseable-internal
248+
depends_on:
249+
- parseable-query
250+
- penguin
251+
- minio
252+
deploy:
253+
restart_policy:
254+
condition: on-failure
255+
delay: 20s
256+
max_attempts: 3

0 commit comments

Comments
 (0)