MAINT: use local docker registry

mgxd · mgxd · commit 3fdb218053bd · 2020-02-19T10:03:36.000-05:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,8 +5,8 @@ jobs:
     environment:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
       SCRATCH: "/scratch"
-    docker:
-      - image: docker:18.01.0-ce-git
+    machine:
+      image: circleci/classic:201711-01
     working_directory: /tmp/src/fmriprep
     steps:
       - checkout
@@ -18,33 +18,46 @@ jobs:
               circleci step halt
             fi
 
-      - run:
-          name: Install parallel gzip and python3
-          command: |
-            apk add --no-cache pigz python3
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
-            - docker-v3-{{ .Branch }}-
-            - docker-v3-master-
-            - docker-v3-
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-master-
+            - build-v4-
           paths:
-            - /tmp/cache/docker.tar.gz
-      - setup_remote_docker
+            - /tmp/docker
+      - run:
+          name: Set up Docker registry
+          command: |
+            docker run -d -p 5000:5000 --restart=always --name=registry \
+                -v /tmp/docker:/var/lib/registry registry:2
       - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
+          name: Pull images
           command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
+            set +e
+            docker pull localhost:5000/ubuntu
+            success=$?
+            set -e
+            if [[ "$success" = "0" ]]; then
+                echo "Pulling from local registry"
+                docker tag localhost:5000/ubuntu ubuntu:xenial-20191010
+                docker pull localhost:5000/fmriprep
+                docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
+                docker tag localhost:5000/dmriprep poldracklab/fmriprep
+            else
+                echo "Pulling from Docker Hub"
+                docker pull ubuntu:xenial-20191010
+                docker tag ubuntu:xenial-20191010 localhost:5000/ubuntu
+                docker push localhost:5000/ubuntu
+                docker pull poldracklab/fmriprep:latest
             fi
       - run:
           name: Build Docker image
           no_output_timeout: 60m
           command: |
+            export PY3=$(pyenv versions | grep '3\.' |
+                         sed -e 's/.* 3\./3./' -e 's/ .*//')
+            pyenv local $PY3
             # Get version, update files.
             THISVERSION=$( python3 get_version.py )
             if [[ ${THISVERSION:0:1} == "0" ]] ; then
@@ -58,52 +71,34 @@ jobs:
             sed -i "s/title = {FMRIPrep}/title = {FMRIPrep ${CIRCLE_TAG:-$THISVERSION}}/" fmriprep/data/boilerplate.bib
             # Build docker image
             e=1 && for i in {1..5}; do
-              docker build \
+              docker build --rm \
                 --cache-from=poldracklab/fmriprep \
-                --rm=false \
                 -t poldracklab/fmriprep:latest \
                 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
                 --build-arg VCS_REF=`git rev-parse --short HEAD` \
                 --build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \
               && e=0 && break || sleep 15
             done && [ "$e" -eq "0" ]
-            # test if development image should be built
-            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[build[ _]?devel\]' )" != "" ]]; then
-              echo "building development container"
-              docker tag poldracklab/fmriprep poldracklab/fmriprep:unstable
-              docker build \
-                --rm=false \
-                -t poldracklab/fmriprep_devel \
-                -f Dockerfile_devel .
-            fi
       - run:
-          name: Smoke test Docker image
+          name: Docker push to local registry
+          no_output_timeout: 40m
           command: |
-            THISVERSION=$( python3 get_version.py )
-            THISVERSION=${THISVERSION%.dirty*}
-            THISVERSION=${CIRCLE_TAG:-$THISVERSION}
-            DOCKERVERSION=$(docker run --rm -it poldracklab/fmriprep:latest --version)
-            DOCKERVERSION=${DOCKERVERSION%$'\r'}
-            DOCKERVERSION=${DOCKERVERSION#*"fMRIPrep v"}
-            echo "VERSION: \"$THISVERSION\""
-            echo "DOCKERVERSION: \"${DOCKERVERSION}\""
-            test "$DOCKERVERSION" = "$THISVERSION"
+            docker tag poldracklab/fmriprep:latest localhost:5000/fmriprep
+            docker push localhost:5000/fmriprep
       - run:
-          name: Docker save
-          no_output_timeout: 40m
+          name: Docker registry garbage collection
           command: |
-            mkdir -p /tmp/cache
-            docker save ubuntu:xenial-20161213 poldracklab/fmriprep:latest \
-            | pigz -3 > /tmp/cache/docker.tar.gz
+            docker exec -it registry /bin/registry garbage-collect --delete-untagged \
+              /etc/docker/registry/config.yml
 
       - persist_to_workspace:
           root: /tmp
           paths:
             - src/fmriprep
       - save_cache:
-         key: docker-v3-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
+         key: build-v4-{{ .Branch }}-{{ epoch }}
          paths:
-            - /tmp/cache/docker.tar.gz
+            - /tmp/docker
 
   get_data:
     machine:
@@ -210,18 +205,24 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-
+          paths:
+            - /tmp/docker
+      - restore_cache:
+          keys:
+            - data-v8-{{ .Revision }}
       - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
+          name: Set-up a Docker registry
           command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              sudo apt update && sudo apt -y install pigz
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
-            fi
+            docker run -d -p 5000:5000 --restart=always --name=registry \
+                -v /tmp/docker:/var/lib/registry registry:2
+      - run:
+          name: Pull images from local registry
+          command: |
+            docker pull localhost:5000/fmriprep
+            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
       - run:
           name: Run fMRIPrep tests
           no_output_timeout: 2h
@@ -323,7 +324,11 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-
+          paths:
+            - /tmp/docker
       - restore_cache:
           keys:
             - data-v7-{{ .Branch }}-{{ .Revision }}
@@ -333,23 +338,22 @@ jobs:
             - ds005-anat-v17-{{ .Branch }}
             - ds005-anat-v17-master
             - ds005-anat-v17-
+      - run:
+          name: Set-up a Docker registry
+          command: |
+            docker run -d -p 5000:5000 --restart=always --name=registry \
+                -v /tmp/docker:/var/lib/registry registry:2
+      - run:
+          name: Pull images from local registry
+          command: |
+            docker pull localhost:5000/fmriprep
+            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
       - run:
           name: Setting up test
           command: |
             mkdir -p /tmp/ds005/derivatives && sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds005/derivatives && sudo setfacl -m group:$(id -gn):rwx /tmp/ds005/derivatives
             pip install --upgrade pip setuptools
             pip install --upgrade /tmp/src/fmriprep/wrapper/
-      - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
-          command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              sudo apt update && sudo apt -y install pigz
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
-            fi
       - run:
           name: Run anatomical workflow on ds005
           no_output_timeout: 2h
@@ -479,7 +483,11 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-
+          paths:
+            - /tmp/docker
       - restore_cache:
           keys:
             - data-v7-{{ .Branch }}-{{ .Revision }}
@@ -495,17 +503,15 @@ jobs:
             mkdir -p /tmp/ds054/derivatives && sudo setfacl -d -m group:$(id -gn):rwx /tmp/ds054/derivatives && sudo setfacl -m group:$(id -gn):rwx /tmp/ds054/derivatives
             pip install --upgrade pip setuptools
             pip install --upgrade /tmp/src/fmriprep/wrapper/
+            name: Set-up a Docker registry
+            command: |
+              docker run -d -p 5000:5000 --restart=always --name=registry \
+                  -v /tmp/docker:/var/lib/registry registry:2
       - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
+          name: Pull images from local registry
           command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              sudo apt update && sudo apt -y install pigz
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
-            fi
+            docker pull localhost:5000/fmriprep
+            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
       - run:
           name: Run anatomical workflow on ds054
           no_output_timeout: 2h
@@ -617,7 +623,11 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-
+          paths:
+            - /tmp/docker
       - restore_cache:
           keys:
             - data-v7-{{ .Branch }}-{{ .Revision }}
@@ -634,16 +644,15 @@ jobs:
             pip install --upgrade pip setuptools
             pip install --upgrade /tmp/src/fmriprep/wrapper/
       - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
+          name: Set-up a Docker registry
           command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              sudo apt update && sudo apt -y install pigz
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
-            fi
+            docker run -d -p 5000:5000 --restart=always --name=registry \
+                -v /tmp/docker:/var/lib/registry registry:2
+      - run:
+          name: Pull images from local registry
+          command: |
+            docker pull localhost:5000/fmriprep
+            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
       - run:
           name: Run anatomical workflow on ds000210
           no_output_timeout: 2h
@@ -712,20 +721,40 @@ jobs:
     working_directory: /tmp/src/fmriprep
     steps:
 
+      - run:
+          name: Check whether image should be deployed to Docker Hub
+          command: |
+            if [[ "$CIRCLE_PROJECT_USERNAME" != "poldracklab" ]]; then
+              echo "Nothing to deploy for $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME."
+              circleci step halt
+            fi
+      - checkout:
+          path: /home/circleci/src/fmriprep
+      - run:
+          name: Check whether build should be skipped
+          command: |
+            cd /home/circleci/src/fmriprep
+            if [[ "$( git log --format='format:%s' -n 1 $CIRCLE_SHA1 | grep -i -E '^docs?(\(\w+\))?:' )" != "" ]]; then
+              echo "Only docs build"
+              circleci step halt
+            fi
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-
+          paths:
+            - /tmp/docker
       - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
+          name: Setup a Docker registry
           command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              sudo apt update && sudo apt -y install pigz
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
-            fi
+            docker run -d -p 5000:5000 --restart=always --name=registry \
+              -v /tmp/docker:/var/lib/registry registry:2
+      - run:
+          name: Pull images from local registry
+          command: |
+            docker pull localhost:5000/fmriprep
+            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
       - run:
           name: Deploy to Docker Hub
           no_output_timeout: 40m
@@ -753,18 +782,21 @@ jobs:
             fi
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ .Revision }}
+            - build-v4-{{ .Branch }}-{{ epoch }}
+            - build-v4-{{ .Branch }}-
+            - build-v4-
+          paths:
+            - /tmp/docker
       - run:
-          name: Load Docker image layer cache
-          no_output_timeout: 30m
+          name: Set-up a Docker registry
           command: |
-            docker info
-            set +o pipefail
-            if [ -f /tmp/cache/docker.tar.gz ]; then
-              sudo apt update && sudo apt -y install pigz
-              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
-              docker images
-            fi
+            docker run -d -p 5000:5000 --restart=always --name=registry \
+                -v /tmp/docker:/var/lib/registry registry:2
+      - run:
+          name: Pull images from local registry
+          command: |
+            docker pull localhost:5000/fmriprep
+            docker tag localhost:5000/fmriprep poldracklab/fmriprep:latest
       - run:
           name: Deploy to Docker Hub
           no_output_timeout: 40m
