Skip to content

Commit ed5bdca

Browse files
rjrudinrjrudin
committed
Changing docker-compose to default to single node
Keeping separate file for 3-node cluster until DBQ-710 is resolved.
1 parent 89b2b78 commit ed5bdca

File tree

5 files changed

+161
-143
lines changed

5 files changed

+161
-143
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@ logs
1616
.ipynb_checkpoints
1717
venv
1818
.venv
19+
docker

‎CONTRIBUTING.md‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,22 @@ have cloned this repository to your local workstation.
55

66
In order to develop and/or test the connector, or to try out the PySpark instructions below, you first
77
need to deploy the test application in this project to MarkLogic. You can do so either on your own installation of
8-
MarkLogic, or you can use `docker-compose` to install a 3-node MarkLogic cluster with a load balancer in front of it.
8+
MarkLogic, or you can use `docker-compose` to install MarkLogic, optionally as a 3-node cluster with a load balancer
9+
in front of it.
910

10-
## Installing a 3-node cluster with docker-compose
11+
## Installing MarkLogic with docker-compose
1112

1213
If you wish to use `docker-compose`, perform the following steps before deploying the test application.
1314

1415
1. [Install Docker](https://docs.docker.com/get-docker/).
1516
2. Ensure that you don't have a MarkLogic instance running locally (if you do, you may run into port conflicts in
1617
the next step).
17-
3. Run `./gradlew dockerUp` (Gradle tasks are included as shortcuts for running Docker commands). This will start up
18-
a 3-node cluster with a load balancer in front of it. Additionally, the 8000/8001/8002 ports are available on the
19-
"bootstrap" node of the cluster for accessing the out-of-the-box MarkLogic applications.
18+
3. Run `docker-compose up -d --build`.
19+
20+
The above will result in a new MarkLogic instance with a single node.
21+
22+
Alternatively, if you would like to test against a 3-node MarkLogic cluster with a load balancer in front of it,
23+
run `docker-compose -f docker-compose-3nodes.yaml up -d --build`.
2024

2125
### Accessing MarkLogic logs in Grafana
2226

‎build.gradle‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,6 @@ task perfTest(type: JavaExec) {
9898
args mlHost
9999
}
100100

101-
task dockerUp(type: Exec) {
102-
description = "Creates and starts a 3 node MarkLogic cluster."
103-
commandLine "docker-compose", "up", "-d", "--build"
104-
}
105-
106101
task dockerBuildCache(type: Exec) {
107102
description = "Creates an image named 'marklogic-spark-cache' containing a cache of the Gradle dependencies."
108103
commandLine 'docker', 'build', '--no-cache', '-t', 'marklogic-spark-cache', '.'

‎docker-compose-3nodes.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
version: '3.8'
2+
name: marklogic_spark_3nodes
3+
4+
services:
5+
6+
# See https://caddyserver.com/docs/quick-starts/reverse-proxy for more information on using Caddy as a reverse proxy
7+
# and load balancer.
8+
caddy-load-balancer:
9+
image: caddy:2-alpine
10+
volumes:
11+
- ./caddy/data:/data
12+
- ./caddy/config/Caddyfile:/etc/caddy/Caddyfile
13+
depends_on:
14+
- bootstrap_3n
15+
- node2
16+
- node3
17+
ports:
18+
# For running marklogic-spark-connector tests against the 3-node cluster
19+
- 8016:8816
20+
- 8015:8815
21+
# For running performance tests against quick-table data
22+
- 8009:8809
23+
# For the getting-started project
24+
- 8020:8820
25+
networks:
26+
- external_net
27+
- internal_net
28+
29+
# Named "bootstrap" as it's the first MarkLogic host created. Other MarkLogic hosts will then join the cluster created
30+
# by this host. Note that each MarkLogic host has its 8000-8002 ports exposed externally so that the apps on those
31+
# ports can each be accessed if needed.
32+
bootstrap_3n:
33+
image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
34+
platform: linux/amd64
35+
container_name: bootstrap_3n
36+
hostname: bootstrap_3n.local
37+
labels:
38+
logging: "promtail"
39+
logging_jobname: "containerlogs"
40+
environment:
41+
- MARKLOGIC_INIT=true
42+
- MARKLOGIC_ADMIN_USERNAME=admin
43+
- MARKLOGIC_ADMIN_PASSWORD=admin
44+
volumes:
45+
- ./docker/marklogic/logs/bootstrap_3n:/var/opt/MarkLogic/Logs
46+
ports:
47+
- 8000-8002:8000-8002
48+
networks:
49+
- external_net
50+
- internal_net
51+
52+
node2:
53+
image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
54+
platform: linux/amd64
55+
container_name: node2
56+
hostname: node2.local
57+
labels:
58+
logging: "promtail"
59+
logging_jobname: "containerlogs"
60+
environment:
61+
- MARKLOGIC_INIT=true
62+
- MARKLOGIC_ADMIN_USERNAME=admin
63+
- MARKLOGIC_ADMIN_PASSWORD=admin
64+
- MARKLOGIC_JOIN_CLUSTER=true
65+
- MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n.local
66+
volumes:
67+
- ./docker/marklogic/logs/node2:/var/opt/MarkLogic/Logs
68+
depends_on:
69+
- bootstrap_3n
70+
ports:
71+
- 8100-8102:8000-8002
72+
networks:
73+
- external_net
74+
- internal_net
75+
76+
node3:
77+
image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
78+
platform: linux/amd64
79+
container_name: node3
80+
hostname: node3.local
81+
labels:
82+
logging: "promtail"
83+
logging_jobname: "containerlogs"
84+
environment:
85+
- MARKLOGIC_INIT=true
86+
- MARKLOGIC_ADMIN_USERNAME=admin
87+
- MARKLOGIC_ADMIN_PASSWORD=admin
88+
- MARKLOGIC_JOIN_CLUSTER=true
89+
- MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n.local
90+
volumes:
91+
- ./docker/marklogic/logs/node3:/var/opt/MarkLogic/Logs
92+
depends_on:
93+
- bootstrap_3n
94+
ports:
95+
- 8200-8202:8000-8002
96+
networks:
97+
- external_net
98+
- internal_net
99+
100+
grafana:
101+
image: grafana/grafana:latest
102+
ports:
103+
- 3000:3000
104+
volumes:
105+
- ./config/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml
106+
environment:
107+
- GF_AUTH_ANONYMOUS_ENABLED=true
108+
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
109+
- GF_AUTH_DISABLE_LOGIN_FORM=true
110+
networks:
111+
- external_net
112+
- internal_net
113+
114+
loki:
115+
image: grafana/loki:latest
116+
ports:
117+
- 3100:3100
118+
command: -config.file=/etc/loki/local-config.yaml
119+
networks:
120+
- external_net
121+
- internal_net
122+
123+
promtail:
124+
image: grafana/promtail:latest
125+
container_name: promtail
126+
volumes:
127+
- ./config/promtail.yaml:/etc/promtail/docker-config.yaml
128+
- ./docker/marklogic/logs:/var/log/marklogic
129+
- /var/lib/docker/containers:/var/lib/docker/containers:ro
130+
- /var/run/docker.sock:/var/run/docker.sock
131+
command: -config.file=/etc/promtail/docker-config.yaml
132+
depends_on:
133+
- loki
134+
- bootstrap_3n
135+
- node2
136+
- node3
137+
networks:
138+
- internal_net
139+
140+
networks:
141+
external_net: { }
142+
internal_net:
143+
internal: true
144+
driver_opts:
145+
com.docker.network.bridge.enable_icc: "true"

‎docker-compose.yaml‎

Lines changed: 6 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -1,145 +1,18 @@
11
version: '3.8'
2-
name: ml_spark
2+
name: marklogic_spark
33

44
services:
55

6-
# See https://caddyserver.com/docs/quick-starts/reverse-proxy for more information on using Caddy as a reverse proxy
7-
# and load balancer.
8-
caddy-load-balancer:
9-
image: caddy:2-alpine
10-
volumes:
11-
- ./caddy/data:/data
12-
- ./caddy/config/Caddyfile:/etc/caddy/Caddyfile
13-
depends_on:
14-
- bootstrap_3n
15-
- node2
16-
- node3
17-
ports:
18-
# For running marklogic-spark-connector tests against the 3-node cluster
19-
- 8016:8816
20-
- 8015:8815
21-
# For running performance tests against quick-table data
22-
- 8009:8809
23-
# For the getting-started project
24-
- 8020:8820
25-
networks:
26-
- external_net
27-
- internal_net
28-
29-
# Named "bootstrap" as it's the first MarkLogic host created. Other MarkLogic hosts will then join the cluster created
30-
# by this host. Note that each MarkLogic host has its 8000-8002 ports exposed externally so that the apps on those
31-
# ports can each be accessed if needed.
32-
bootstrap_3n:
33-
image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
34-
platform: linux/amd64
35-
container_name: bootstrap_3n
36-
hostname: bootstrap_3n.local
37-
labels:
38-
logging: "promtail"
39-
logging_jobname: "containerlogs"
40-
environment:
41-
- MARKLOGIC_INIT=true
42-
- MARKLOGIC_ADMIN_USERNAME=admin
43-
- MARKLOGIC_ADMIN_PASSWORD=admin
44-
volumes:
45-
- ./logs/bootstrap_3n:/var/opt/MarkLogic/Logs
46-
ports:
47-
- 8000-8002:8000-8002
48-
networks:
49-
- external_net
50-
- internal_net
51-
52-
node2:
53-
image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
54-
platform: linux/amd64
55-
container_name: node2
56-
hostname: node2.local
57-
labels:
58-
logging: "promtail"
59-
logging_jobname: "containerlogs"
60-
environment:
61-
- MARKLOGIC_INIT=true
62-
- MARKLOGIC_ADMIN_USERNAME=admin
63-
- MARKLOGIC_ADMIN_PASSWORD=admin
64-
- MARKLOGIC_JOIN_CLUSTER=true
65-
- MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n.local
66-
volumes:
67-
- ./logs/node2:/var/opt/MarkLogic/Logs
68-
depends_on:
69-
- bootstrap_3n
70-
ports:
71-
- 8100-8102:8000-8002
72-
networks:
73-
- external_net
74-
- internal_net
75-
76-
node3:
6+
marklogic:
777
image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0"
788
platform: linux/amd64
79-
container_name: node3
80-
hostname: node3.local
81-
labels:
82-
logging: "promtail"
83-
logging_jobname: "containerlogs"
849
environment:
8510
- MARKLOGIC_INIT=true
8611
- MARKLOGIC_ADMIN_USERNAME=admin
8712
- MARKLOGIC_ADMIN_PASSWORD=admin
88-
- MARKLOGIC_JOIN_CLUSTER=true
89-
- MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n.local
9013
volumes:
91-
- ./logs/node3:/var/opt/MarkLogic/Logs
92-
depends_on:
93-
- bootstrap_3n
94-
ports:
95-
- 8200-8202:8000-8002
96-
networks:
97-
- external_net
98-
- internal_net
99-
100-
grafana:
101-
image: grafana/grafana:latest
14+
- ./docker/marklogic/logs:/var/opt/MarkLogic/Logs
10215
ports:
103-
- 3000:3000
104-
volumes:
105-
- ./config/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml
106-
environment:
107-
- GF_AUTH_ANONYMOUS_ENABLED=true
108-
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
109-
- GF_AUTH_DISABLE_LOGIN_FORM=true
110-
networks:
111-
- external_net
112-
- internal_net
113-
114-
loki:
115-
image: grafana/loki:latest
116-
ports:
117-
- 3100:3100
118-
command: -config.file=/etc/loki/local-config.yaml
119-
networks:
120-
- external_net
121-
- internal_net
122-
123-
promtail:
124-
image: grafana/promtail:latest
125-
container_name: promtail
126-
volumes:
127-
- ./config/promtail.yaml:/etc/promtail/docker-config.yaml
128-
- ./logs:/var/log/marklogic
129-
- /var/lib/docker/containers:/var/lib/docker/containers:ro
130-
- /var/run/docker.sock:/var/run/docker.sock
131-
command: -config.file=/etc/promtail/docker-config.yaml
132-
depends_on:
133-
- loki
134-
- bootstrap_3n
135-
- node2
136-
- node3
137-
networks:
138-
- internal_net
139-
140-
networks:
141-
external_net: { }
142-
internal_net:
143-
internal: true
144-
driver_opts:
145-
com.docker.network.bridge.enable_icc: "true"
16+
- "8000-8002:8000-8002"
17+
- "8015:8015"
18+
- "8016:8016"

0 commit comments

Comments
 (0)