Merge pull request #17 from geoadmin/task_BGDIINF_SB-1839_migration_dockerhub_to_ecr

BGDIINF_SB-1839: Migration from dockerhub to ecr

Author: rebert

Makefile

@@ -39,16 +39,21 @@ ISORT_CMD := $(PIPENV_RUN) isort
 NOSE_CMD := $(PIPENV_RUN) nose2
 PYLINT_CMD := $(PIPENV_RUN) pylint
 
-# Docker variables
-DOCKER_IMG_LOCAL_TAG = swisstopo/$(SERVICE_NAME):local
+# AWS variables
+AWS_DEFAULT_REGION = eu-central-1
 
 # Docker metadata
 GIT_HASH := `git rev-parse HEAD`
+GIT_HASH_SHORT = `git rev-parse --short HEAD`
 GIT_BRANCH := `git symbolic-ref HEAD --short 2>/dev/null`
 GIT_DIRTY := `git status --porcelain`
 GIT_TAG := `git describe --tags || echo "no version info"`
 AUTHOR := $(USER)
 
+# Docker variables
+DOCKER_REGISTRY = 974517877189.dkr.ecr.eu-central-1.amazonaws.com
+DOCKER_IMG_LOCAL_TAG = $(DOCKER_REGISTRY)/$(SERVICE_NAME):local-$(USER)-$(GIT_HASH_SHORT)
+
 all: help
 
 # This bit check define the build/python "target": if the system has an acceptable version of python, there will be no need to install python locally.
@@ -86,7 +91,9 @@ help:
 	@echo -e " \033[1mLOCAL SERVER TARGETS\033[0m "
 	@echo "- serve              Run the project using the flask debug server"
 	@echo "- gunicornserve      Run the project using the gunicorn WSGI server"
+	@echo "- dockerlogin        Login to the AWS ECR registery for pulling/pushing docker images"
 	@echo "- dockerbuild        Build the project localy using the gunicorn WSGI server inside a container"
+	@echo "- dockerpush         Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))"
 	@echo "- dockerrun          Run the project using the gunicorn WSGI server inside a container. (Exposed_port: $(HTTP_PORT)"
 	@echo "- shutdown           Stop the aforementioned container"
 	@echo -e " \033[1mCLEANING TARGETS\033[0m "
@@ -143,6 +150,10 @@ gunicornserve: $(REQUIREMENTS_TIMESTAMP)
 	${PYTHON_CMD} wsgi.py
 
 # Docker related functions.
+.PHONY: dockerlogin
+dockerlogin:
+	aws --profile swisstopo-bgdi-builder ecr get-login-password --region $(AWS_DEFAULT_REGION) | docker login --username AWS --password-stdin $(DOCKER_REGISTRY)
+
 .PHONY: dockerbuild
 dockerbuild:
 	docker build \
@@ -152,6 +163,10 @@ dockerbuild:
 		--build-arg VERSION="$(GIT_TAG)" \
 		--build-arg AUTHOR="$(AUTHOR)" -t $(DOCKER_IMG_LOCAL_TAG) .
 
+.PHONY: dockerpush
+dockerpush: dockerbuild
+	docker push $(DOCKER_IMG_LOCAL_TAG)
+
 export-http-port:
 	@export HTTP_PORT=$(HTTP_PORT)
 

README.md

@@ -11,7 +11,7 @@
 - [Dependencies](#dependencies)
 - [Service API](#service-api)
 - [Local Development](#local-development)
-- [Docker](#docker)
+- [Docker helpers](#docker-helpers)
 - [Versioning](#versioning)
 - [Deployment](#deployment)
 
@@ -123,17 +123,12 @@ To stop serving through containers,
     make shutdown
 
 Is the command you're looking for.
+### Docker helpers
 
-## Docker
+From each github PR that is merged into `master` or into `develop`, one Docker image is built and pushed on AWS ECR with the following tag:
 
-The service is encapsulated in a Docker image. Images are pushed on the public [Dockerhub](https://hub.docker.com/r/swisstopo/service-shortlink/tags) registry. From each github PR that is merged into develop branch, one Docker image is built and pushed with the following tags:
-
-- `develop.latest`
-- `CURRENT_VERSION-beta.INCREMENTAL_NUMBER`
-
-From each github PR that is merged into master, one Docker image is built an pushed with the following tag:
-
-- `VERSION`
+- `vX.X.X` for tags on master
+- `vX.X.X-beta.X` for tags on develop 
 
 Each image contains the following metadata:
 
@@ -143,12 +138,16 @@ Each image contains the following metadata:
 - git.dirty
 - version
 
-These metadata can be seen directly on the dockerhub registry in the image layers or can be read with the following command
+These metadata can be read with the following command
 
 ```bash
+# NOTE: Currently we don't have permission to do docker pull on AWS ECR
+make dockerlogin
+docker pull 974517877189.dkr.ecr.eu-central-1.amazonaws.com/service-shortcut:develop.latest
+
 # NOTE: jq is only used for pretty printing the json output,
 # you can install it with `apt install jq` or simply enter the command without it
-docker image inspect --format='{{json .Config.Labels}}' swisstopo/service-shortlink:develop.latest | jq
+docker image inspect --format='{{json .Config.Labels}}' 974517877189.dkr.ecr.eu-central-1.amazonaws.com/service-shortcut:develop.latest | jq
 ```
 
 You can also check these metadata on a running container as follows
@@ -157,11 +156,20 @@ You can also check these metadata on a running container as follows
 docker ps --format="table {{.ID}}\t{{.Image}}\t{{.Labels}}"
 ```
 
-## Versioning
+To build a local docker image tagged as `service-shortcut:local-${USER}-${GIT_HASH_SHORT}` you can
+use
+
+```bash
+make dockerbuild
+```
+
+To push the image on the ECR repository use the following two commands
 
-This service uses [SemVer](https://semver.org/) as versioning scheme. The versioning is automatically handled by `.github/workflows/main.yml` file.
+```bash
+make dockerlogin
+make dockerpush
+```
 
-See also [Git Flow - Versioning](https://github.com/geoadmin/doc-guidelines/blob/master/GIT_FLOW.md#versioning) for more information on the versioning guidelines.
 
 ## Deployment
 
@@ -181,4 +189,4 @@ The service is configured by Environment Variable:
 | ALLOWED_DOMAINS | 'admin.ch,swisstopo.ch,bgdi.ch' | A comma separated list of allowed domains names |
 | ALLOWED_HOSTS | 'api.geo.admin.ch,api3.geo.admin.ch' | a comma separated list of allowed hostnames |
 | AWS_DYNAMODB_TABLE_NAME | 'shortlinks_test' | The dynamodb table name |
-| AWS_DYNAMODB_TABLE_REGION | 'eu-central-1' | The AWS region in which the table is hosted. |
+| AWS_DYNAMODB_TABLE_REGION | 'eu-central-1' | The AWS region in which the table is hosted. |

buildspec.yml

@@ -2,24 +2,29 @@ version: 0.2
 
 env:
   variables:
-    IMAGE_BASE_NAME: 'swisstopo/service-shortlink'
+    IMAGE_BASE_NAME: 'service-shortlink'
+    REGISTRY: '974517877189.dkr.ecr.eu-central-1.amazonaws.com'
     SHELL: /bin/bash
     AWS_DEFAULT_REGION: eu-central-1
     USER: "aws_code_build"
     TEST_REPORT_DIR: "./tests/report"
     TEST_REPORT_FILE: "nose2-junit.xml"
-  parameter-store:
-    CI_DOCKERHUB_USER: "/dockerhub/user"
-    CI_DOCKERHUB_PASSWORD: "/dockerhub/password"
 
 phases:
   install:
     runtime-versions:
       docker: 18
     commands:
       - echo "Installing necessary softwares"
-      - docker login -u ${CI_DOCKERHUB_USER} -p ${CI_DOCKERHUB_PASSWORD}
-      - apt-get update && apt-get install -y docker-compose python3-pip
+      - apt-get update && apt-get install -y docker-compose python3-pip pass gnupg2
+      - echo "Install aws cli v2 for docker login to ECR registry"
+      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      - unzip awscliv2.zip
+      - ./aws/install
+      - aws --version
+      - echo "Login to AWS ECR docker registry"
+      - aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${REGISTRY}
+
   pre_build:
     commands:
       - echo "export of the image tag for build and push purposes"
@@ -39,11 +44,11 @@ phases:
   build:
     commands:
       - echo Build started on $(date)
-      - export DOCKER_IMG_TAG=${IMAGE_BASE_NAME}:${GITHUB_TAG}
-      - export DOCKER_IMG_TAG_LATEST=${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.latest
+      - export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_TAG}
+      - export DOCKER_IMG_TAG_LATEST=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.latest
       - |-
         if [ "${GITHUB_TAG}" = "" ] ; then
-          export DOCKER_IMG_TAG=${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.${GITHUB_COMMIT}
+          export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH}.${GITHUB_COMMIT}
           export GITHUB_TAG=${GITHUB_COMMIT}
         fi
       - echo "Building docker image with tags ${DOCKER_IMG_TAG} and ${DOCKER_IMG_TAG_LATEST}"
@@ -69,7 +74,7 @@ phases:
           docker push ${DOCKER_IMG_TAG}
         fi
       - |
-        if [ "${GITHUB_BRANCH}" = "develop" ]; then
+        if [ "${GITHUB_BRANCH}" = "develop" ] || [ "${GITHUB_BRANCH}" = "master" ]; then
           echo "Pushing ${DOCKER_IMG_TAG} and ${DOCKER_IMG_TAG_LATEST} to dockerhub"
           docker push ${DOCKER_IMG_TAG}
           docker push ${DOCKER_IMG_TAG_LATEST}