Skip to content

Commit c152a66

Browse files
sandereggsanderegg
authored
Simplify release process (#1748)
use tagging instead of staging branch
1 parent 493be5b commit c152a66

21 files changed

+497
-571
lines changed

‎.github/workflows/ci-hotfix.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Github-CI-Hotfix-Release
2+
3+
on:
4+
push:
5+
tags:
6+
- v[0-9]+.[0-9]+.[0-9]+
7+
8+
env:
9+
# secrets can be set in settings/secrets on github
10+
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
11+
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
12+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
13+
14+
jobs:
15+
deploy:
16+
if: contains(github.event.base_ref, 'refs/heads/hotfix_v')
17+
# only run on staging branch
18+
name: deploy release
19+
runs-on: ubuntu-latest
20+
env:
21+
FROM_TAG_PREFIX: hotfix-github
22+
TO_TAG_PREFIX: release-github
23+
steps:
24+
- uses: actions/checkout@v2
25+
- name: setup docker
26+
run: |
27+
sudo ./ci/github/helpers/setup_docker_compose.bash
28+
./ci/github/helpers/setup_docker_experimental.bash
29+
./ci/github/helpers/setup_docker_buildx.bash
30+
echo ::set-env name=DOCKER_BUILDX::1
31+
- name: set owner variable
32+
run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
33+
- name: set git tag
34+
run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
35+
- name: deploy
36+
run: ./ci/deploy/dockerhub-tag-version.bash

‎.github/workflows/ci-release.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,13 @@ env:
1313

1414
jobs:
1515
deploy:
16+
if: github.event.base_ref == 'refs/heads/master'
1617
# only run on staging branch
17-
if: github.event.base_ref == 'refs/heads/staging'
1818
name: deploy release
1919
runs-on: ubuntu-latest
2020
env:
21-
TAG_PREFIX: staging-github
21+
FROM_TAG_PREFIX: staging-github
22+
TO_TAG_PREFIX: release-github
2223
steps:
2324
- uses: actions/checkout@v2
2425
- name: setup docker
@@ -32,4 +33,4 @@ jobs:
3233
- name: set git tag
3334
run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
3435
- name: deploy
35-
run: ./ci/deploy/dockerhub-release.bash
36+
run: ./ci/deploy/dockerhub-tag-version.bash

‎.github/workflows/ci-staging.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Github-CI-Staging
2+
3+
on:
4+
push:
5+
tags:
6+
- staging_[a-zA-Z]+[0-9]+
7+
8+
env:
9+
# secrets can be set in settings/secrets on github
10+
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
11+
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
12+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
13+
14+
jobs:
15+
deploy:
16+
if: github.event.base_ref == 'refs/heads/master'
17+
name: deploy staging
18+
runs-on: ubuntu-latest
19+
env:
20+
FROM_TAG_PREFIX: master-github
21+
TO_TAG_PREFIX: staging-github
22+
steps:
23+
- uses: actions/checkout@v2
24+
- name: setup docker
25+
run: |
26+
sudo ./ci/github/helpers/setup_docker_compose.bash
27+
./ci/github/helpers/setup_docker_experimental.bash
28+
./ci/github/helpers/setup_docker_buildx.bash
29+
echo ::set-env name=DOCKER_BUILDX::1
30+
- name: set owner variable
31+
run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
32+
- name: set git tag
33+
run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
34+
- name: deploy
35+
run: ./ci/deploy/dockerhub-tag-version.bash

‎.github/workflows/ci-testing-deploy.yml‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,19 @@ name: Github-CI Push/PR
22

33
on:
44
push:
5+
branches:
6+
- "*"
7+
tags-ignore:
8+
- "*"
9+
paths-ignore:
10+
- "*.md"
11+
- "*.png"
12+
- "*.svg"
13+
- "docs/**"
14+
- ".vscode/**"
515
pull_request:
16+
branches:
17+
- "*"
618
paths-ignore:
719
- "*.md"
820
- "*.png"
@@ -1176,17 +1188,8 @@ jobs:
11761188
env:
11771189
TAG_PREFIX: master-github
11781190
run: ./ci/deploy/dockerhub-deploy.bash
1179-
- name: deploy staging
1180-
if: github.ref == 'refs/heads/staging'
1191+
- name: deploy hotfix
1192+
if: contains(github.ref, 'refs/heads/hotfix_v')
11811193
env:
1182-
TAG_PREFIX: staging-github
1194+
TAG_PREFIX: hotfix-github
11831195
run: ./ci/deploy/dockerhub-deploy.bash
1184-
# TODO: here we need a incoming webhook for mattermost (ask admin for it) (https://docs.mattermost.com/developer/webhooks-incoming.html)
1185-
# - name: mattermost notification
1186-
# uses: tferreira/matterfy@releases/v1
1187-
# if: always()
1188-
# with:
1189-
# type: ${{ job.status }}
1190-
# job_name: '*Simcore build state*'
1191-
# channel: '${{ secrets.MATTERMOST_CHANNEL }}'
1192-
# url: ${{ secrets.MATTERMOST_URL }}

‎.travis.yml‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -404,32 +404,35 @@ jobs:
404404
# deployment to staging environ ----------------------------------------------------------------
405405
- stage: deployment
406406
name: staging
407-
if: branch = staging
408407
env:
409408
- OWNER=${TRAVIS_REPO_SLUG%/*}
410-
- TAG_PREFIX=staging
409+
- FROM_TAG_PREFIX=master-travis
410+
- TO_TAG_PREFIX=staging-travis
411+
- GIT_TAG=${TRAVIS_TAG}
411412
git:
412413
depth: false
413414
script: echo "Deploy staging"
414415
deploy:
415416
- provider: script
416-
script: unbuffer bash ci/deploy/dockerhub-deploy.bash
417+
script: unbuffer bash ci/deploy/dockerhub-tag-version.bash
417418
on:
418-
branch: staging
419-
419+
all_branches: true
420+
tags: true
421+
condition: $TRAVIS_TAG =~ ^FREEZE_[a-zA-Z]+[0-9]+$
420422
# deployment to production environ ----------------------------------------------------------------
421423
- stage: deployment
422424
name: production/release
423425
env:
424426
- OWNER=${TRAVIS_REPO_SLUG%/*}
425-
- TAG_PREFIX=staging
427+
- FROM_TAG_PREFIX=staging-travis
428+
- TO_TAG_PREFIX=release-travis
426429
- GIT_TAG=${TRAVIS_TAG}
427430
git:
428431
depth: false
429432
script: echo "Deploy production version $TRAVIS_TAG"
430433
deploy:
431434
- provider: script
432-
script: unbuffer bash ci/deploy/dockerhub-release.bash
435+
script: unbuffer bash ci/deploy/dockerhub-tag-version.bash
433436
on:
434437
all_branches: true
435438
tags: true

‎Makefile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,3 +523,44 @@ clean-all: clean clean-more clean-images # Deep clean including .venv and produc
523523
.PHONY: reset
524524
reset: ## restart docker daemon (LINUX ONLY)
525525
sudo systemctl restart docker
526+
527+
528+
# RELEASE --------------------------------------------------------------------------------------------------------------------------------------------
529+
530+
staging_prefix := staging_
531+
prod_prefix := v
532+
_git_get_current_branch = $(shell git rev-parse --abbrev-ref HEAD)
533+
# NOTE: be careful that GNU Make replaces newlines with space which is why this command cannot work using a Make function
534+
_url_encoded_title = $(if $(findstring -staging, $@),Staging%20$(name),)$(version)
535+
_url_encoded_tag = $(if $(findstring -staging, $@),$(staging_prefix)$(name),$(prod_prefix))$(version)
536+
_url_encoded_target = $(if $(git_sha),$(git_sha),$(if $(findstring -hotfix, $@),$(_git_get_current_branch),master))
537+
define _url_encoded_logs
538+
$(shell \
539+
scripts/url-encoder.bash \
540+
"$$(git log \
541+
$$(git describe --match="$(if $(findstring -staging, $@),$(staging_prefix),$(prod_prefix))*" --abbrev=0 --tags)..$(if $(git_sha),$(git_sha),HEAD) \
542+
--pretty=format:"- %s")"\
543+
)
544+
endef
545+
_git_get_repo_orga_name = $(shell git config --get remote.origin.url | \
546+
grep --perl-regexp --only-matching "((?<=git@github\.com:)|(?<=https:\/\/github\.com\/))(.*?)(?=.git)")
547+
548+
.PHONY: .check-master-branch
549+
.check-master-branch:
550+
@if [ "$(_git_get_current_branch)" != "master" ]; then\
551+
echo -e "\e[91mcurrent branch is not master branch."; exit 1;\
552+
fi
553+
554+
.PHONY: release-staging release-prod
555+
release-staging release-prod: .check-master-branch ## Helper to create a staging or production release in Github (usage: make release-staging name=sprint version=1 git_sha=optional or make release-prod version=1.2.3 git_sha=optional)
556+
# ensure tags are uptodate
557+
@git pull --tags
558+
@echo -e "\e[33mOpen the following link to create the $(if $(findstring -staging, $@),staging,production) release:";
559+
@echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring -staging, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)";
560+
561+
.PHONY: release-hotfix
562+
release-hotfix: ## Helper to create a hotfix release in Github (usage: make release-hotfix version=1.2.4 git_sha=optional)
563+
# ensure tags are uptodate
564+
@git pull --tags
565+
@echo -e "\e[33mOpen the following link to create the $(if $(findstring -staging, $@),staging,production) release:";
566+
@echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring -staging, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)";

‎ci/README.md‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,20 @@
11
# Continuous Integration
22

3-
## Travis / Dockerhub setup
3+
## CI / Dockerhub setup
44

5-
The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg).
5+
The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) and Gihub-CI [workflows](.github/workflows) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg) and the [Hotfix Workflow](docs/img/git-hotfix-workflow.svg).
66

7-
To this end Travis should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by Travis.
7+
### Configuration
8+
Github Actions and/or Travis-CI should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by the CI.
89

9-
Configuring your travis settings will speed up the travis CI process by making use of Dockerhub (or another docker registry) to move docker images between stages.
10+
Configuring your CI settings will speed up the CI process by making use of Dockerhub (or another docker registry) to move docker images between stages.
1011

11-
### Travis configuration
12+
### Secrets configuration
1213

1314
Define the following secure environment variables in your fork:
1415

1516
```bash
1617
DOCKER_REGISTRY # this shall be set to your own dockerhub repository account for example: itisfoundation
17-
DOCKER_USERNAME # the docker username (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml))
18-
DOCKER_PASSWORD # the docker password (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml))
18+
DOCKER_USERNAME # the docker username
19+
DOCKER_PASSWORD # the docker password
1920
```
20-
21-
### Release workflow
22-
23-
![Git release workflow](../docs/img/git-release-workflow.svg)

‎ci/deploy/dockerhub-deploy.bash‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,40 @@
1-
#!/bin/bash
2-
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
3-
set -euo pipefail
1+
#!/usr/bin/env bash
2+
# strict mode
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
46
IFS=$'\n\t'
57

6-
bash ci/helpers/dockerhub_login.bash
8+
my_dir="$(dirname "$0")"
9+
source "$my_dir/../../scripts/helpers/logger.bash"
710

811
# check script needed variables
912
if [ ! -v TAG_PREFIX ]; then
10-
echo "## ERROR: incorrect use of script. TAG_PREFIX (e.g. master, staging) not defined!"
11-
exit 1
13+
error_exit "$LINENO" "incorrect use of script. TAG_PREFIX (e.g. master, staging) not defined!"
1214
fi
1315

16+
log_info "logging in dockerhub..."
17+
bash ci/helpers/dockerhub_login.bash
18+
1419
# pull the current tested build
20+
1521
DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
1622
export DOCKER_IMAGE_TAG
23+
log_info "pulling build ${DOCKER_IMAGE_TAG}"
1724
make pull-version tag-local
1825

1926
# show current images on system
20-
echo "## Before push"
27+
log_info "Before push"
2128
make info-images
2229

2330
# re-tag build
2431
DOCKER_IMAGE_TAG="$TAG_PREFIX-latest"
2532
export DOCKER_IMAGE_TAG
33+
log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
2634
make push-version
2735

2836
# re-tag build to master-github-DATE.GIT_SHA
2937
DOCKER_IMAGE_TAG=$TAG_PREFIX-$(date --utc +"%Y-%m-%d--%H-%M").$(git rev-parse HEAD)
3038
export DOCKER_IMAGE_TAG
39+
log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
3140
make push-version
32-
33-
# show update of images on system
34-
echo "## After push"
35-
make info-images

‎ci/deploy/dockerhub-release.bash‎

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎ci/deploy/dockerhub-tag-version.bash‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/usr/bin/env bash
2+
# strict mode
3+
set -o errexit # abort on nonzero exitstatus
4+
set -o nounset # abort on unbound variable
5+
set -o pipefail # don't hide errors within pipes
6+
IFS=$'\n\t'
7+
8+
my_dir="$(dirname "$0")"
9+
source "$my_dir/../../scripts/helpers/logger.bash"
10+
11+
# check script needed variables
12+
if [ ! -v FROM_TAG_PREFIX ] || [ ! -v TO_TAG_PREFIX ] || [ ! -v GIT_TAG ]; then
13+
error_exit "$LINENO" "incorrect use of script. FROM_TAG_PREFIX/TO_TAG_PREFIX (e.g. master, staging), and/or GIT_TAG not defined!"
14+
fi
15+
16+
log_info "logging in dockerhub..."
17+
bash ci/helpers/dockerhub_login.bash
18+
19+
log_info "finding the version in the docker hub registry..."
20+
# find and pull the tagged build
21+
# find the docker image tag
22+
export ORG=${DOCKER_REGISTRY}
23+
export REPO="webserver"
24+
# FROM_TAG_PREFIX-DATE.GIT_SHA
25+
export TAG_PATTERN="^${FROM_TAG_PREFIX}-.+\..+"
26+
DOCKER_IMAGE_TAG=$(./ci/helpers/find_docker_image_tag_from_git_sha.bash | awk 'END{print}') || exit $?
27+
log_info "found image ${DOCKER_IMAGE_TAG}"
28+
export DOCKER_IMAGE_TAG
29+
log_info "pulling images ${DOCKER_IMAGE_TAG} from ${DOCKER_REGISTRY}"
30+
make pull-version tag-local
31+
32+
# show current images on system
33+
log_info "Before push"
34+
make info-images
35+
36+
# re-tag images to ${TO_TAG_PREFIX}-{GIT_TAG}-DATE.GIT_SHA
37+
readonly GIT_COMMIT_SHA=$(git show-ref -s "${GIT_TAG}")
38+
DOCKER_IMAGE_TAG="${TO_TAG_PREFIX}-${GIT_TAG}"-$(date --utc +"%Y-%m-%d--%H-%M")."${GIT_COMMIT_SHA}"
39+
export DOCKER_IMAGE_TAG
40+
log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
41+
make push-version
42+
43+
# push latest image ${TO_TAG_PREFIX}-latest
44+
DOCKER_IMAGE_TAG="${TO_TAG_PREFIX}-latest"
45+
export DOCKER_IMAGE_TAG
46+
log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
47+
make push-version
48+
49+
log_info "complete!"

0 commit comments

Comments
 (0)