Skip to content

Commit 3ee4447

Browse files
lautermanlauterman
authored
Merge pull request #215 from psu-libraries/preview/pushwait
Enhance CI/CD pipeline for Docker image management
2 parents c9c6f59 + 940ebdf commit 3ee4447

File tree

2 files changed

+129
-51
lines changed

2 files changed

+129
-51
lines changed

‎.circleci/config.yml‎

Lines changed: 117 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,32 @@ parameters:
2222
CI_UTILS_IMAGE_TAG:
2323
type: string
2424
default: "v4.3.0"
25+
CONFIG_REPO_NAME:
26+
type: string
27+
default: "pdfapi-config"
28+
CONFIG_REPO_URL:
29+
type: string
30+
default: "git@github.com:psu-libraries/pdfapi-config.git"
31+
REGISTRY_HOST:
32+
type: string
33+
default: "harbor.k8s.libraries.psu.edu"
34+
CLUSTERS_DEV:
35+
type: string
36+
default: "uldev"
37+
CLUSTERS_PROD:
38+
type: string
39+
default: "prod"
2540

2641
commands:
2742

28-
clone-pdfapi-config:
43+
clone-config-repo:
2944
steps:
3045
- add_ssh_keys
3146
- run:
3247
name: Prepare GitHub Access
3348
command: |
3449
ssh-keyscan github.com > ~/.ssh/known_hosts
35-
git clone git@github.com:psu-libraries/pdfapi-config.git
50+
git clone $CONFIG_REPO_URL
3651
3752
compute-docker-tag:
3853
steps:
@@ -53,20 +68,24 @@ executors:
5368
parameters:
5469
image_tag:
5570
type: string
71+
registry_host:
72+
type: string
5673
docker:
57-
- image: harbor.k8s.libraries.psu.edu/library/ci-utils:<< parameters.image_tag >>
74+
- image: << parameters.registry_host >>/library/ci-utils:<< parameters.image_tag >>
5875

5976
jobs:
6077

61-
build-and-push:
78+
build-image:
6279
environment:
63-
REGISTRY_HOST: harbor.k8s.libraries.psu.edu
80+
REGISTRY_HOST: << pipeline.parameters.REGISTRY_HOST >>
6481
REGISTRY_REPO: "library/$CIRCLE_PROJECT_REPONAME"
6582
executor:
6683
name: ci-utils
6784
image_tag: << pipeline.parameters.CI_UTILS_IMAGE_TAG >>
85+
registry_host: << pipeline.parameters.REGISTRY_HOST >>
6886
steps:
69-
- setup_remote_docker
87+
- setup_remote_docker:
88+
docker_layer_caching: true
7089
- checkout
7190
- run:
7291
name: Checkout Branch
@@ -88,6 +107,42 @@ jobs:
88107
tag: $TAG
89108
extra_build_args: "--target production"
90109
use-buildkit: true
110+
- run:
111+
name: Save Docker image
112+
command: |
113+
docker save $REGISTRY_HOST/library/$CIRCLE_PROJECT_REPONAME:$TAG -o /tmp/docker-image.tar
114+
- run:
115+
name: Save image info for push job
116+
command: |
117+
mkdir -p /tmp/docker-image-info
118+
echo "$REGISTRY_HOST/library/$CIRCLE_PROJECT_REPONAME:$TAG" > /tmp/docker-image-info/image-name.txt
119+
- persist_to_workspace:
120+
root: /tmp
121+
paths:
122+
- docker-image.tar
123+
- docker-image-info
124+
125+
push-image:
126+
environment:
127+
REGISTRY_HOST: << pipeline.parameters.REGISTRY_HOST >>
128+
REGISTRY_REPO: "library/$CIRCLE_PROJECT_REPONAME"
129+
executor:
130+
name: ci-utils
131+
image_tag: << pipeline.parameters.CI_UTILS_IMAGE_TAG >>
132+
registry_host: << pipeline.parameters.REGISTRY_HOST >>
133+
steps:
134+
- setup_remote_docker:
135+
docker_layer_caching: false
136+
- checkout
137+
- compute-docker-tag
138+
- attach_workspace:
139+
at: /tmp
140+
- run:
141+
name: Load Docker image
142+
command: |
143+
docker load -i /tmp/docker-image.tar
144+
- docker/check:
145+
registry: $REGISTRY_HOST
91146
- docker/push:
92147
image: $REGISTRY_REPO
93148
registry: $REGISTRY_HOST
@@ -100,85 +155,88 @@ jobs:
100155
executor:
101156
name: ci-utils
102157
image_tag: << pipeline.parameters.CI_UTILS_IMAGE_TAG >>
158+
registry_host: << pipeline.parameters.REGISTRY_HOST >>
103159
environment:
104-
CONFIG_REPO: git@github.com:psu-libraries/pdfapi-config.git
160+
CONFIG_REPO_NAME: << pipeline.parameters.CONFIG_REPO_NAME >>
161+
CONFIG_REPO_URL: << pipeline.parameters.CONFIG_REPO_URL >>
105162
DELETE_BRANCH: << parameters.delete_branch >>
106163
steps:
107-
- clone-pdfapi-config
164+
- clone-config-repo
108165
- run:
109166
name: Delete Deployment
110167
command: |
111168
export TRIGGERED_BY="$CIRCLE_USERNAME"
112-
cd pdfapi-config
113-
./bin/delete-deployment
169+
cd $CONFIG_REPO_NAME
170+
./bin/ci-delete-manifest
114171
115172
deploy-application:
116173
executor:
117174
name: ci-utils
118175
image_tag: << pipeline.parameters.CI_UTILS_IMAGE_TAG >>
176+
registry_host: << pipeline.parameters.REGISTRY_HOST >>
119177
environment:
120-
CONFIG_REPO: git@github.com:psu-libraries/pdfapi-config.git
121-
steps:
122-
- clone-pdfapi-config
123-
- compute-docker-tag
124-
- run:
125-
name: Generate Application
126-
command: |
127-
export TRIGGERED_BY="$CIRCLE_USERNAME"
128-
cd pdfapi-config
129-
./bin/generate_application
130-
131-
deploy-preview-app:
132-
executor:
133-
name: ci-utils
134-
image_tag: << pipeline.parameters.CI_UTILS_IMAGE_TAG >>
135-
environment:
136-
CONFIG_REPO: git@github.com:psu-libraries/pdfapi-config.git
178+
clusters: << parameters.clusters >>
179+
CONFIG_REPO_NAME: << pipeline.parameters.CONFIG_REPO_NAME >>
180+
CONFIG_REPO_URL: << pipeline.parameters.CONFIG_REPO_URL >>
181+
parameters:
182+
clusters:
183+
type: string
137184
steps:
138-
- clone-pdfapi-config
185+
- clone-config-repo
139186
- compute-docker-tag
140187
- run:
141-
name: Generate Preview Application
188+
name: Add or Update Application Manifest
142189
command: |
143190
export TRIGGERED_BY="$CIRCLE_USERNAME"
144-
cd pdfapi-config
145-
./bin/deploy-preview-app
191+
# Split comma-separated clusters into array and loop
192+
clusters="${clusters// /}"
193+
IFS=',' read -ra CLUSTERS \<<< "$clusters"
194+
echo "Pre loop Directory: $(pwd)"
195+
for cluster in "${CLUSTERS[@]}"; do
196+
export TARGET_CLUSTER="$cluster"
197+
echo "Processing cluster: $TARGET_CLUSTER"
198+
cd $CONFIG_REPO_NAME
199+
./bin/ci-write-manifest
200+
echo "Finished processing $TARGET_CLUSTER"
201+
git status
202+
done
146203
147204
release-image:
148205
parameters:
149-
to_tag:
206+
clusters:
150207
type: string
151208
from_tag:
152209
type: string
210+
to_tag:
211+
type: string
153212
executor:
154213
name: ci-utils
155214
image_tag: << pipeline.parameters.CI_UTILS_IMAGE_TAG >>
215+
registry_host: << pipeline.parameters.REGISTRY_HOST >>
156216
environment:
157-
CONFIG_REPO: git@github.com:psu-libraries/pdfapi-config.git
158-
REGISTRY_HOST: harbor.k8s.libraries.psu.edu
217+
clusters: << parameters.clusters >>
159218
FROM_TAG: << parameters.from_tag >>
160219
TO_TAG: << parameters.to_tag >>
220+
CONFIG_REPO_NAME: << pipeline.parameters.CONFIG_REPO_NAME >>
221+
CONFIG_REPO_URL: << pipeline.parameters.CONFIG_REPO_URL >>
222+
REGISTRY_HOST: << pipeline.parameters.REGISTRY_HOST >>
161223
steps:
162-
- clone-pdfapi-config
224+
- clone-config-repo
163225
- run:
164226
name: Tag & Release Image
165227
command: |
166228
export TRIGGERED_BY="$CIRCLE_USERNAME"
167229
export REGISTRY_REPO="$CIRCLE_PROJECT_REPONAME"
168230
/usr/local/bin/image-tag
169-
cd pdfapi-config
231+
cd $CONFIG_REPO_NAME
170232
/usr/local/bin/image-release-pr clusters/prod/manifests/prod.yaml
171233
172234
test-application:
173235
docker:
174236
- image: cimg/ruby:3.4.1
175237
environment:
176-
RAILS_ENV: test
177-
MYSQL_PASSWORD: pdf_accessibility_api
178-
REGISTRY_HOST: harbor.k8s.libraries.psu.edu
179-
REGISTRY_REPO: library/pdf_accessibility_api
180238
GITHUB_USER: 'psu-stewardship-bot'
181-
CONFIG_REPO: git@github.com:psu-libraries/pdfapi-config.git
239+
MYSQL_PASSWORD: pdf_accessibility_api
182240
steps:
183241
- checkout
184242
- setup_remote_docker:
@@ -191,7 +249,7 @@ jobs:
191249
bundle _$(grep -A 1 'BUNDLED WITH' Gemfile.lock | tail -n 1 | xargs)_ install
192250
- run:
193251
name: Build and run containers
194-
command: docker-compose -f docker-compose.yml build --no-cache && docker-compose -f docker-compose.yml up --force-recreate -d
252+
command: docker-compose -f docker-compose.yml build && docker-compose -f docker-compose.yml up --force-recreate -d
195253
- run:
196254
name: Wait for mysql to be ready
197255
command: docker-compose exec web bash ./wait_for_db.sh
@@ -200,7 +258,7 @@ jobs:
200258
command: docker-compose exec -T web bundle exec niftany
201259
- run:
202260
name: "Run tests"
203-
command: RAILS_ENV=test docker-compose exec -it web bundle exec rspec
261+
command: RAILS_ENV=test docker-compose exec -T web bundle exec rspec
204262

205263
workflows:
206264

@@ -216,12 +274,12 @@ workflows:
216274
branches:
217275
only: main
218276

219-
docker_push:
277+
docker-push:
220278
unless:
221279
equal: [ "delete", << pipeline.parameters.GHA_Event >> ]
222280
jobs:
223-
- build-and-push:
224-
name: build-and-push
281+
- build-image:
282+
name: build-image
225283
context:
226284
- org-global
227285
filters:
@@ -233,26 +291,29 @@ workflows:
233291
name: deploy-application
234292
context:
235293
- org-global
294+
clusters: << pipeline.parameters.CLUSTERS_DEV >>
236295
filters:
237296
branches:
238297
only:
239298
- main
299+
- /preview\/.*/
240300
requires:
241-
- build-and-push
242-
- test-application
243-
- deploy-preview-app:
244-
name: deploy-preview-app
301+
- push-image
302+
- push-image:
303+
name: push-image
245304
context:
246305
- org-global
247306
filters:
248307
branches:
249308
only:
309+
- main
250310
- /preview\/.*/
251311
requires:
252-
- build-and-push
312+
- build-image
253313
- test-application
254314
- release-image:
255315
name: release-image
316+
clusters: << pipeline.parameters.CLUSTERS_PROD >>
256317
context:
257318
- org-global
258319
from_tag: "<< pipeline.git.revision >>"
@@ -266,3 +327,8 @@ workflows:
266327
name: test-application
267328
context:
268329
- org-global
330+
filters:
331+
branches:
332+
only:
333+
- main
334+
- /preview\/.*/

‎README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,15 @@ To run the tests within the container:
8888
## Swagger Docs
8989

9090
Our API and webhook documentation is generated using RSwag and the RSwag DSL from the spec files in `spec/requests/api/v1/api-docs`. If you make changes to the RSwag spec files, run `RAILS_ENV=test bundle exec rails rswag` to regenerate the swagger.yaml.
91+
92+
93+
## Preview deployments
94+
Preview deployments are triggered off branch-name conventions. To trigger a Preview deployment, create a branch with a `preview/` prefix, and CI will generate a corresponding `Application` object that ArgoCD will deploy
95+
96+
Example: we are fixing a bug in something
97+
98+
`git checkout -b preview/fixbug`
99+
100+
Will deploy application pdfapi-fixbug-uldev, and it will be available via pdfapi-fixbug-uldev.uldev.k8s.libraries.psu.edu
101+
102+
Preview branch names shouldn't contain underscores or special characters (they need to be DNS complient)

0 commit comments

Comments
 (0)