Merge pull request docker-archive#470 from docker/aws_vacuum

Added swarm wide vacuum job to cleanup unused resources

Author: mavenugo

aws/cloudformation/docker_for_aws.json

@@ -44,7 +44,14 @@
             "Default" : "3",
             "AllowedValues" : ["1", "3", "5"],
             "Description" : "Number of Swarm manager nodes (1, 3, 5)"
+        },
+        "EnableSystemPrune" : {
+            "Type" : "String",
+            "Default" : "no",
+            "AllowedValues" : ["no", "yes"],
+            "Description" : "Cleans up unused images, containers, networks and volumes"
         }
+
     },
     "Metadata" : {
         "AWS::CloudFormation::Interface" : {
@@ -55,15 +62,16 @@
                 },
                 {
                     "Label" : { "default":"Swarm Properties" },
-                    "Parameters" : [ "ManagerInstanceType", "InstanceType", "KeyName" ]
+                    "Parameters" : [ "ManagerInstanceType", "InstanceType", "KeyName", "EnableSystemPrune" ]
                 }
             ],
             "ParameterLabels" : {
                 "ManagerSize" : { "default" : "Number of Swarm managers?" },
                 "ClusterSize" : { "default" : "Number of Swarm worker nodes?" },
                 "InstanceType" : { "default" : "Agent worker instance type?" },
                 "ManagerInstanceType" : { "default" : "Swarm manager instance type?" },
-                "KeyName" : { "default" : "Which SSH key to use?" }
+                "KeyName" : { "default" : "Which SSH key to use?" },
+                "EnableSystemPrune": {"default" : "Enable daily resource cleanup?"}
             }
         }
     },
@@ -602,6 +610,7 @@
                                 "-e ACCOUNT_ID='",{ "Ref" : "AWS::AccountId" }, "' ",
                                 "-e SWARM_QUEUE='",{ "Ref" : "SwarmSQS" }, "' ",
                                 "-e CLEANUP_QUEUE='",{ "Ref" : "SwarmSQSCleanup" }, "' ",
+                                "-e RUN_VACUUM='",{ "Ref" : "EnableSystemPrune" }, "' ",
                                 "-e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION ",
                                 "-v /var/run/docker.sock:/var/run/docker.sock ",
                                 "-v /usr/bin/docker:/usr/bin/docker ",
@@ -739,6 +748,7 @@
                                 "-e ACCOUNT_ID='",{ "Ref" : "AWS::AccountId" }, "' ",
                                 "-e SWARM_QUEUE='",{ "Ref" : "SwarmSQS" }, "' ",
                                 "-e CLEANUP_QUEUE='",{ "Ref" : "SwarmSQSCleanup" }, "' ",
+                                "-e RUN_VACUUM='",{ "Ref" : "EnableSystemPrune" }, "' ",
                                 "-e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION ",
                                 "-v /var/run/docker.sock:/var/run/docker.sock ",
                                 "-v /usr/bin/docker:/usr/bin/docker ",

aws/dockerfiles/Dockerfile.guide

@@ -18,10 +18,11 @@ ADD files/guide/crontab.txt /usr/docker/crontab.txt
 ADD files/guide/watcher.sh /usr/bin
 ADD files/guide/cleanup.sh /usr/bin
 ADD files/guide/buoy.sh /usr/bin
+ADD files/guide/vacuum.sh /usr/bin
 ADD files/bin/buoy /usr/bin
 COPY files/guide/entry.sh /entry.sh
 RUN chmod 755 /usr/bin/watcher.sh /entry.sh /usr/bin/cleanup.sh \
-    /usr/bin/buoy /usr/bin/buoy.sh
+    /usr/bin/buoy /usr/bin/buoy.sh /usr/bin/vacuum.sh
 RUN /usr/bin/crontab /usr/docker/crontab.txt
 
 CMD ["/entry.sh"]

aws/dockerfiles/files/guide/crontab.txt

@@ -2,3 +2,4 @@
 */1 * * * * /usr/bin/watcher.sh >> /var/log/docker/watcher.log
 */5 * * * * /usr/bin/cleanup.sh >> /var/log/docker/cleanup.log
 24 * * * * /usr/bin/buoy.sh >> /var/log/docker/buoy.log
+42 * * * * /usr/bin/vacuum.sh >> /var/log/docker/vacuum.log

aws/dockerfiles/files/guide/vacuum.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# default to no.
+RUN_VACUUM=${RUN_VACUUM:-"no"}
+
+if [[ "$RUN_VACUUM" != "yes" ]] ; then
+    exit 0
+fi
+
+# sleep for a random amount of time, so that we don't run this at the same time on all nodes.
+sleep $[ ( $RANDOM % 240 )  + 1 ]
+
+docker system prune --force