diff --git a/.env b/.env new file mode 100644 index 0000000..50accaa --- /dev/null +++ b/.env @@ -0,0 +1 @@ +MONGO_VERSION=4.0 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 249cda9..e69de29 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +0,0 @@ -/data \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ae2fb56 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +.PHONY: initiate help + +help: ## print this message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' + +up: ## Start Mongo cluster (without init) + docker-compose up -d + +init: ## Launch mongo cluster init + ./initiate + +clean: ## Stop docker containers and clean volumes + docker-compose down -v + +start: up init ## Start containers and launch init + +stop: ## Stop docker containers + docker-compose stop + +watch: ## Watch logs + docker-compose logs -f + +restart: clean start ## Clean and restart + diff --git a/README.md b/README.md index 9af2ebb..359e080 100644 --- a/README.md +++ b/README.md @@ -3,53 +3,63 @@ This repository provides a fully sharded mongo environment using docker-compose The MongoDB environment consists of the following docker containers - - **mongosrs(1-3)n(1-3)**: Mongod data server with three replica sets containing 3 nodes each (9 containers) - - **mongocfg(1-3)**: Stores metadata for sharded data distribution (3 containers) - - **mongos(1-2)**: Mongo routing service to connect to the cluster through (1 container) + - **mongors(1-2)n(1-3)**: Mongod data server with two replica sets containing 3 nodes each (2 replica + 1 arbiter * 2 : 6 containers) + - **mongocfg(1-3)**: Stores metadata for sharded data distribution CSRS (3 containers) + - **mongos(1-2)**: Mongo routing service to connect to the cluster through (2 containers) ## Caveats - This is designed to have a minimal disk footprint at the cost of durability. - This is designed in no way for production but as a cheap learning and exploration vehicle. -## Installation (Debian base): - -### Install Docker - - sudo apt-get install -y apparmor lxc cgroup-lite curl - wget -qO- https://get.docker.com/ | sh - sudo usermod -aG docker YourUserNameHere - sudo service docker restart - -### Install Docker-compose (1.4.2+) - - sudo su - curl -L https://github.com/docker/compose/releases/download/1.4.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - exit +## Installation ### Check out the repository git clone git@github.com:singram/mongo-docker-compose.git cd mongo-docker-compose - ### Setup Cluster -This will pull all the images from [Docker index](https://index.docker.io/u/jacksoncage/mongo/) and run all the containers. +This will pull all the images from [Docker index](https://hub.docker.com/_/mongo) and run all the containers. - docker-compose up + make up -Please note that you will need docker-compose 1.4.2 or better for this to work due to circular references between cluster members. You will need to run the following *once* only to initialize all replica sets and shard data across them - ./initiate + make init + +Or both : + + make start + +Makefile documentation : + + ⇒ make help + + ----- BUILD ------------------------------------------------------------------------------ + up Start docker-compose + init Launch mongo cluster init + clean Stop docker containers and clean volumes + start Start container and init + stop Stop docker containers + watch Watch logs + restart Clean and restart + ----- OTHERS ----------------------------------------------------------------------------- + help print this message + You should now be able connect to mongos1 and the new sharded cluster from the mongos container itself using the mongo shell to connect to the running mongos process docker exec -it mongos1 mongo --port 21017 +### Version + +By default, it will start a MongoDb 4.0 cluster, but you can also start MongoDB in version 3.6 : + + MONGO_VERSION=3.6 make start + ## Persistent storage -Data is stored at `./data/` and is excluded from version control. Data will be persistent between container runs. To remove all data `./reset` +Data is stored in docker volumes. To remove all data : `make clean`. ## TODO @@ -64,3 +74,4 @@ Data is stored at `./data/` and is excluded from version control. Data will be p - [Mongo Docker ](https://github.com/jacksoncage/mongo-docker) - [DnsDock](https://github.com/tonistiigi/dnsdock) - [Docker](https://github.com/dotcloud/docker/) + - [WaitForIt](https://github.com/vishnubob/wait-for-it) diff --git a/data/mongo-cfg-1/.gitkeep b/data/mongo-cfg-1/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongo-cfg-2/.gitkeep b/data/mongo-cfg-2/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongo-cfg-3/.gitkeep b/data/mongo-cfg-3/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongors1n1/.gitkeep b/data/mongors1n1/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongors1n2/.gitkeep b/data/mongors1n2/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongors1n3/.gitkeep b/data/mongors1n3/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongors2n1/.gitkeep b/data/mongors2n1/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongors2n2/.gitkeep b/data/mongors2n2/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/mongors2n3/.gitkeep b/data/mongors2n3/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docker-compose.yml b/docker-compose.yml index cebbba9..0284cff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,125 +1,97 @@ -version: "2" +version: "3" services: mongors1n1: - container_name: mongors1n1 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --replSet mongors1 --dbpath /data/db --nojournal --oplogSize 16 --noauth + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --replSet mongors1 --shardsvr --oplogSize 16 --noauth --bind_ip_all environment: TERM: xterm - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongors1n1:/data/db + expose: + - 27018 mongors1n2: - container_name: mongors1n2 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --replSet mongors1 --dbpath /data/db --nojournal --oplogSize 16 --noauth + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --replSet mongors1 --shardsvr --oplogSize 16 --noauth --bind_ip_all environment: TERM: xterm - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongors1n2:/data/db + expose: + - 27018 mongors1n3: - container_name: mongors1n3 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --replSet mongors1 --dbpath /data/db --nojournal --oplogSize 16 --noauth + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --replSet mongors1 --shardsvr --oplogSize 16 --noauth --bind_ip_all environment: TERM: xterm - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongors1n3:/data/db + expose: + - 27018 mongors2n1: - container_name: mongors2n1 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --replSet mongors2 --dbpath /data/db --nojournal --oplogSize 16 --noauth + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --replSet mongors2 --shardsvr --oplogSize 16 --noauth --bind_ip_all environment: TERM: xterm - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongors2n1:/data/db + expose: + - 27018 mongors2n2: - container_name: mongors2n2 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --replSet mongors2 --dbpath /data/db --nojournal --oplogSize 16 --noauth + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --replSet mongors2 --shardsvr --oplogSize 16 --noauth --bind_ip_all environment: TERM: xterm - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongors2n2:/data/db + expose: + - 27018 mongors2n3: - container_name: mongors2n3 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --replSet mongors2 --dbpath /data/db --nojournal --oplogSize 16 --noauth + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --replSet mongors2 --shardsvr --oplogSize 16 --noauth --bind_ip_all environment: TERM: xterm - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongors2n3:/data/db + expose: + - 27018 mongocfg1: - container_name: mongocfg1 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --dbpath /data/db --configsvr --noauth --port 27017 + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --configsvr --replSet mongocfg1 --noauth --bind_ip_all environment: TERM: xterm expose: - - "27017" - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongocfg1:/data/db + - 27019 mongocfg2: - container_name: mongocfg2 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --dbpath /data/db --configsvr --noauth --port 27017 + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --configsvr --replSet mongocfg1 --noauth --bind_ip_all environment: TERM: xterm expose: - - "27017" - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongocfg2:/data/db + - 27019 mongocfg3: - container_name: mongocfg3 - image: mongo:3.0.14 - command: mongod --noprealloc --smallfiles --dbpath /data/db --configsvr --noauth --port 27017 + image: mongo:${MONGO_VERSION} + command: mongod --noprealloc --smallfiles --configsvr --replSet mongocfg1 --noauth --bind_ip_all environment: TERM: xterm expose: - - "27017" - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DATA_DIR}/mongocfg3:/data/db + - 27019 mongos1: - container_name: mongos1 - image: mongo:3.0.14 + image: mongo:${MONGO_VERSION} depends_on: - mongocfg1 - mongocfg2 - mongocfg3 - command: mongos --configdb mongocfg1:27017,mongocfg2:27017,mongocfg3:27017 --port 27017 + command: mongos --configdb mongocfg1/mongocfg1:27019,mongocfg2:27019,mongocfg3:27019 --bind_ip_all ports: - 27017:27017 volumes: - - /etc/localtime:/etc/localtime:ro + - ./wait-for-it.sh:/tmp/wait-for-it.sh + mongos2: - container_name: mongos2 - image: mongo:3.0.14 + image: mongo:${MONGO_VERSION} depends_on: - mongocfg1 - mongocfg2 - mongocfg3 - command: mongos --configdb mongocfg1:27017,mongocfg2:27017,mongocfg3:27017 --port 27017 + command: mongos --configdb mongocfg1/mongocfg1:27019,mongocfg2:27019,mongocfg3:27019 --bind_ip_all ports: - - 27018:27017 - volumes: - - /etc/localtime:/etc/localtime:ro - + - 27018:27017 \ No newline at end of file diff --git a/initiate b/initiate index f81e804..1685fe4 100755 --- a/initiate +++ b/initiate @@ -1,13 +1,39 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + for (( rs = 1; rs < 3; rs++ )); do - echo "Intializing replica ${rs} set" - replicate="rs.initiate(); sleep(1000); cfg = rs.conf(); cfg.members[0].host = \"mongors${rs}n1\"; rs.reconfig(cfg); rs.add(\"mongors${rs}n2\"); rs.add(\"mongors${rs}n3\"); rs.status();" - docker exec -it mongors${rs}n1 bash -c "echo '${replicate}' | mongo" + echo "" + echo "=> Intializing replicaset ${rs}" + echo "" + for (( node = 1; node < 3; node++ )); do + docker-compose exec mongos1 bash -c "/tmp/wait-for-it.sh -t 60 mongors${rs}n${node}:27018" + done + replicate="rs.initiate(); sleep(1000); cfg = rs.conf(); cfg.members[0].host = \"mongors${rs}n1:27018\"; sleep(1000);rs.reconfig(cfg); rs.add(\"mongors${rs}n2:27018\"); rs.addArb(\"mongors${rs}n3:27018\"); rs.status();" + + echo "=> Replicaset${rs} configuration : " + docker-compose exec mongors${rs}n1 bash -c "echo '${replicate}' | mongo --port 27018" done -sleep 2 +echo "" +echo "=> Intializing replicaset for config" +echo "" +replicate="rs.initiate(); sleep(1000); cfg = rs.conf(); cfg.members[0].host = \"mongocfg1:27019\"; rs.reconfig(cfg); rs.add(\"mongocfg2:27019\"); rs.add(\"mongocfg3:27019\"); rs.status();" + +echo "" +echo "=> Config replicaset configuration : " +echo "" +docker-compose exec mongocfg1 bash -c "echo '${replicate}' | mongo --port 27019" + + +docker-compose exec mongos1 bash -c "/tmp/wait-for-it.sh mongocfg1:27019;/tmp/wait-for-it.sh mongocfg2:27019;/tmp/wait-for-it.sh mongocfg1:27019;" + # Add better mechanisum to wait for mongos connectivity to be # established by tailing docker log for connection readiness - -docker exec -it mongos1 bash -c "echo \"sh.addShard('mongors1/mongors1n1:27017'); sh.addShard('mongors2/mongors2n1:27017');\" | mongo " +echo "" +echo "Intializing shards through router" +echo "" +docker-compose exec mongos1 bash -c "/tmp/wait-for-it.sh -t 60 mongos1:27017;/tmp/wait-for-it.sh mongos2:27017" +docker-compose exec mongos1 bash -c "echo \"sh.addShard('mongors1/mongors1n1:27018,mongors1n2:27018'); sh.addShard('mongors2/mongors2n1:27018,mongors2n2:27018');sh.status()\" | mongo " diff --git a/reset b/reset deleted file mode 100755 index 32e8e49..0000000 --- a/reset +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -dirs=( mongocfg1 mongocfg2 mongocfg3 mongors1n1 mongors1n2 mongors1n3 mongors2n1 mongors2n2 mongors2n3 ) -for d in ${dirs[@]}; do - echo "Resetting data/$d" - sudo rm -rf data/$d - mkdir data/$d - touch data/$d/.gitkeep -done diff --git a/wait-for-it.sh b/wait-for-it.sh new file mode 100755 index 0000000..401a6f1 --- /dev/null +++ b/wait-for-it.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + if [[ $ISBUSY -eq 1 ]]; then + nc -z $HOST $PORT + result=$? + else + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + fi + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI="$@" + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-15} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +# check to see if timeout is from busybox? +# check to see if timeout is from busybox? +TIMEOUT_PATH=$(realpath $(which timeout)) +if [[ $TIMEOUT_PATH =~ "busybox" ]]; then + ISBUSY=1 + BUSYTIMEFLAG="-t" +else + ISBUSY=0 + BUSYTIMEFLAG="" +fi + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec $CLI +else + exit $RESULT +fi