+ Implemented real cluster deploy

+ Renamed Docker image

Author: Genarito

docker-compose.yml

@@ -2,49 +2,36 @@ version: "3.6"
 services:
   # Master
   master-node:
-    image: "jwaresolutions/bigdata-cluster"
+    image: "jwaresolutions/big-data-cluster"
     restart: "always"
-    # command: bash -c "echo 'n' | /usr/local/hadoop/bin/hdfs namenode -format &&
     command: bash -c "/home/big_data/spark-cmd.sh start master-node"
     networks:
-      - spark-net
+      - cluster-net
     volumes:
       # - "./data:/home/big_data/data" # Your data
       - hdfs-master-data:/home/hadoop/data/nameNode
       - hdfs-master-checkpoint-data:/home/hadoop/data/namesecondary
-    deploy:
-      placement:
-        # set node labels using docker node update --label-add key=value <NODE ID> from swarm manager
-        constraints:
-          - node.labels.role==master
 
   # Workers
   worker:
-    image: "jwaresolutions/bigdata-cluster"
+    image: "jwaresolutions/big-data-cluster"
     restart: "always"
     command: bash -c "/home/big_data/spark-cmd.sh start"
     depends_on:
       - "master-node"
     volumes:
       - hdfs-worker-data:/home/hadoop/data/dataNode
-    deploy:
-      placement:
-        # set node labels using `docker node update --label-add key=value <NODE ID>` from swarm manager
-        constraints:
-          - node.labels.role==worker
-      # Deploy 3 containers for this service
-      replicas: 3
     networks:
-      - spark-net
+      - cluster-net
 
 volumes:
   hdfs-master-data:
   hdfs-master-checkpoint-data:
   hdfs-worker-data:
 
-# Create the spark-net network
+# Create the cluster-net network
 networks:
-  spark-net:
+  cluster-net:
     name: "cluster_net" # Useful for format as it does not allow '-' char on command
     driver: bridge
     attachable: false # Attachable: true prevents user to connect to Hadoop panels

docker-compose_cluster.yml

@@ -0,0 +1,67 @@
+version: "3.6"
+services:
+  # Master
+  master-node:
+    image: "jwaresolutions/big-data-cluster"
+    command: bash -c "/home/big_data/spark-cmd.sh start master-node"
+    ports:
+      - target: 8088
+        published: 8088
+        protocol: tcp
+        mode: host
+      - target: 8080
+        published: 8080
+        protocol: tcp
+        mode: host
+      - target: 9870
+        published: 9870
+        protocol: tcp
+        mode: host
+    networks:
+      - cluster-net
+    volumes:
+      # - "./data:/home/big_data/data" # Your data
+      - hdfs-master-data:/home/hadoop/data/nameNode
+      - hdfs-master-checkpoint-data:/home/hadoop/data/namesecondary
+    deploy:
+      mode: global # Required by Docker Swarm to make published ports work with other services
+      endpoint_mode: dnsrr # Required to prevent 
+      placement:
+        # Set node labels using `docker node update --label-add role=master <NODE ID>` from swarm manager
+        constraints:
+          - node.labels.role==master
+
+  # Workers
+  worker:
+    image: "jwaresolutions/big-data-cluster"
+    command: bash -c "/home/big_data/spark-cmd.sh start"
+    depends_on:
+      - "master-node"
+    volumes:
+      - hdfs-worker-data:/home/hadoop/data/dataNode
+    deploy:
+      placement:
+        # Set node labels using `docker node update --label-add role=worker <NODE ID>` from swarm manager
+        constraints:
+          - node.labels.role==worker
+      # Deploy N containers for this service
+      replicas: 3
+    networks:
+      - cluster-net
+
+volumes:
+  hdfs-master-data:
+    external:
+      name: 'hdsf_master_data_swarm'
+  hdfs-master-checkpoint-data:
+    external:
+      name: 'hdsf_master_checkpoint_data_swarm'
+  hdfs-worker-data:
+    external:
+      name: 'hdsf_worker_data_swarm'
+
+# Uses cluster-net network
+networks:
+  cluster-net:
+    external: true
+    name: cluster_net_swarm

cluster.sh renamed to toy-cluster.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-imageName="jwaresolutions/bigdata-cluster"
+imageName="jwaresolutions/big-data-cluster"
 
 # Bring the services up
 function startServices {
@@ -35,6 +35,11 @@ if [[ $1 = "stop" ]]; then
   exit
 fi
 
+if [[ $1 = "remove" ]]; then
+  docker rm master-node worker-1 worker-2 worker-3
+  exit
+fi
+
 if [[ $1 = "deploy" ]]; then
   docker container rm -f `docker ps -a | grep $imageName | awk '{ print $1 }'` # delete old containers
   docker network rm cluster_net
@@ -62,5 +67,6 @@ fi
 echo "Usage: cluster.sh deploy|start|stop"
 echo "    deploy - create a new Docker network, containers (a master and 3 workers) and start these last"
 echo "    start  - start the existing containers"
-echo "    stop   - stop the running containers" 
-echo "    info   - useful URLs" 
+echo "    stop   - stop the running containers"
+echo "    remove - remove all the created containers"
+echo "    info   - useful URLs"