add S3 storage to config

Author: robkooper

docker-compose.override.example.yml

@@ -9,9 +9,62 @@ services:
     ports:
       - 5672:5672
 
+  # ----------------------------------------------------------------------
+  # DOCKER MANAGEMENT APPLICATION
+  # ----------------------------------------------------------------------
+
+  # Allow to see all docker containers running, restart and see log files. (optional)
+  portainer:
+    image: portainer/portainer:latest
+    command:
+      - --admin-password=${PORTAINER_PASSWORD:-}
+      - --host=unix:///var/run/docker.sock
+    restart: unless-stopped
+    networks:
+      - clowder
+    labels:
+      - "traefik.enable=true"
+      - "traefik.backend=portainer"
+      - "traefik.frontend.rule=${TRAEFIK_HOST:-}PathPrefixStrip: /portainer"
+      - "traefik.website.frontend.whiteList.sourceRange=${TRAEFIK_IPFILTER:-172.16.0.0/12}"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - portainer:/data
+
+  # ----------------------------------------------------------------------
+  # S3 STORAGE CONTAINER
+  # ----------------------------------------------------------------------
+
+  # S3 storage
+  minio:
+    image: minio/minio:latest
+    command: server /data
+    restart: unless-stopped
+    networks:
+      - clowder
+    environment:
+      - MINIO_ACCESS_KEY=${S3_ACCESS_KEY:-clowder}
+      - MINIO_SECRET_KEY=${S3_SECRET_KEY:-catsarecute}
+    labels:
+      - "traefik.enable=true"
+      - "traefik.backend=minio"
+      - "traefik.port=9000"
+      - "traefik.frontend.rule=${TRAEFIK_FRONTEND_RULE:-}PathPrefix:/minio/"
+    volumes:
+      - minio-data:/data
+
+# ----------------------------------------------------------------------
+# VOLUMES FOR PERSISTENT STORAGE
+# ----------------------------------------------------------------------
 volumes:
+  portainer:
+  minio-data:
+    driver_opts:
+      type: none
+      device: '${PWD}/volumes/minio'
+      o: bind
   mongo:
     driver_opts:
       type: none
-      device: '${HOME}/docker/mongo'
+      device: '${PWD}/volumes/mongo'
       o: bind

docker-compose.yml

@@ -45,28 +45,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - traefik:/config
 
-  # ----------------------------------------------------------------------
-  # DOCKER MANAGEMENT APPLICATION
-  # ----------------------------------------------------------------------
-
-  # Allow to see all docker containers running, restart and see log files. (optional)
-  portainer:
-    image: portainer/portainer:latest
-    command:
-      - --admin-password=${PORTAINER_PASSWORD:-}
-      - --host=unix:///var/run/docker.sock
-    restart: unless-stopped
-    networks:
-      - clowder
-    labels:
-      - "traefik.enable=true"
-      - "traefik.backend=portainer"
-      - "traefik.frontend.rule=${TRAEFIK_HOST:-}PathPrefixStrip: /portainer"
-      - "traefik.website.frontend.whiteList.sourceRange=${TRAEFIK_IPFILTER:-172.16.0.0/12}"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - portainer:/data
-
   # ----------------------------------------------------------------------
   # CLOWDER APPLICATION
   # ----------------------------------------------------------------------
@@ -171,7 +149,6 @@ networks:
 # ----------------------------------------------------------------------
 volumes:
   traefik:
-  portainer:
   clowder-data:
   clowder-custom:
   mongo:

docker/custom.conf

@@ -35,7 +35,21 @@ smtp.host=${?SMTP_SERVER}
 
 # storage
 service.byteStorage=services.filesystem.DiskByteStorageService
+service.byteStorage=${?CLOWDER_STORAGE}
+
+# location in case of services.filesystem.DiskByteStorageService
 clowder.diskStorage.path="/home/clowder/data"
+clowder.diskStorage.path=${?CLOWDER_DISKPATH}
+
+# location in case of services.s3.S3ByteStorageService
+clowder.s3.serviceEndpoint="http://minio:9000"
+clowder.s3.bucketName=${?S3_ENDPOINT}
+clowder.s3.bucketName="clowder"
+clowder.s3.bucketName=${?S3_BUCKET}
+clowder.s3.accessKey="clowder"
+clowder.s3.accessKey=${?S3_ACCESS_KEY}
+clowder.s3.secretKey="catsarecute"
+clowder.s3.secretKey=${?S3_SECRET_KEY}
 
 # mongo server
 mongodbURI="mongodb://mongo:27017/clowder"

env.example

@@ -64,12 +64,26 @@
 #SMTP_SERVER=smtp
 
 # ----------------------------------------------------------------------
-# PORTAINER CONFIGURATION
+# BYTE STORAGE
 # ----------------------------------------------------------------------
 
-# password for portainer admin account
-# use docker run --rm httpd:2.4-alpine htpasswd -nbB admin <password> | cut -d ":" -f 2
-#PORTAINER_PASSWORD=$2y$05$5meDPBtS3NNxyGhBpYceVOxmFhiiC3uY5KEy2m0YRbWghhBr2EVn2
+# backend to store the actual bytes:
+# - services.filesystem.DiskByteStorageService
+# - services.s3.S3ByteStorageService
+#CLOWDER_STORAGE=services.filesystem.DiskByteStorageService
+
+# Storage for services.s3.S3ByteStorageService
+#CLOWDER_DISKPATH=/home/clowder/data
+
+# S3 endpoint to use, this assumes a local instance of minio
+#S3_ENDPOINT=http://minio:9000
+
+# bucket that holds the data
+#S3_BUCKET=clowder
+
+# access to the bucket
+#S3_ACCESS_KEY=clowder
+#S3_SECRET_KEY=catsarecute
 
 # ----------------------------------------------------------------------
 # RABBITMQ CONFIGURATION
@@ -86,4 +100,4 @@
 #RABBITMQ_EXCHANGE=clowder
 
 # in case of external rabbitmq, the url to clowder
-#RABBITMQ_CLOWDERURL=https://clowder-docker.ncsa.illinois.edu/clowder/
+#RABBITMQ_CLOWDERURL=https://clowder-docker.ncsa.illinois.edu/clowder/