Skip to content

Commit 9fcd851

Browse files
emmacasolinemmacasolin
committed
ci: review and enhancements
- Removed `rules` from child pipelines and other jobs with dependencies - Separated value for `parallel` into a variable for easier editing - Uploading of the dist separated into its own job so multiple parallel jobs aren't uploading it - Removed `--runInBand` since we want to make use of multiple cores Relates to #65
1 parent f450a32 commit 9fcd851

File tree

3 files changed

+49
-57
lines changed

3 files changed

+49
-57
lines changed

‎.gitlab-ci.yml‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ check:nix-dry:
7474

7575
check:test-generate:
7676
stage: check
77+
needs: []
7778
script:
7879
- mkdir -p ./tmp
7980
- >
@@ -102,12 +103,6 @@ check:test:
102103
- artifact: tmp/test-pipelines.yml
103104
job: check:test-generate
104105
strategy: depend
105-
rules:
106-
# Runs on staging commits and ignores version commits
107-
- if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
108-
# Manually run on commits other than master and staging and ignore version commits
109-
- if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
110-
when: manual
111106

112107
build:merge:
113108
stage: build
@@ -137,7 +132,7 @@ build:merge:
137132
# Runs on tag pipeline where the tag is a prerelease or release version
138133
- if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
139134

140-
build:build-generate:
135+
build:platforms-generate:
141136
stage: build
142137
script:
143138
- mkdir -p ./tmp
@@ -172,10 +167,28 @@ build:platforms:
172167
# Runs on tag pipeline where the tag is a prerelease or release version
173168
- if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
174169

170+
build:dist:
171+
stage: build
172+
needs: []
173+
script:
174+
- >
175+
nix-shell --run '
176+
npm run build --verbose;
177+
'
178+
artifacts:
179+
when: always
180+
paths:
181+
- ./dist
182+
rules:
183+
# Runs on staging commits and ignores version commits
184+
- if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
185+
# Runs on tag pipeline where the tag is a prerelease or release version
186+
- if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
187+
175188
build:prerelease:
176189
stage: build
177190
needs:
178-
- build:platforms
191+
- build:dist
179192
# Don't interrupt publishing job
180193
interruptible: false
181194
before_script:
@@ -198,7 +211,7 @@ build:prerelease:
198211
integration:builds:
199212
stage: integration
200213
needs:
201-
- build:platforms
214+
- build:dist
202215
script:
203216
- mkdir -p ./builds
204217
- >
@@ -395,7 +408,7 @@ integration:merge:
395408
stage: integration
396409
needs:
397410
- build:merge
398-
- job: build:platforms
411+
- job: build:dist
399412
optional: true
400413
- job: integration:nix
401414
optional: true
@@ -471,7 +484,7 @@ release:deployment:tag:
471484
release:distribution:
472485
stage: release
473486
needs:
474-
- build:platforms
487+
- build:dist
475488
- integration:builds
476489
- integration:merge
477490
- release:deployment:tag

‎scripts/build-pipelines.sh‎

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,9 @@ shopt -s nullglob
55

66
# Quote the heredoc to prevent shell expansion
77
cat << "EOF"
8-
workflow:
9-
rules:
10-
# Disable merge request pipelines
11-
- if: $CI_MERGE_REQUEST_ID
12-
when: never
13-
- when: always
148
default:
159
interruptible: true
10+
1611
variables:
1712
GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
1813
GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
@@ -27,6 +22,7 @@ variables:
2722
TS_CACHED_TRANSPILE_PORTABLE: "true"
2823
# Homebrew cache only used by macos runner
2924
HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
25+
3026
# Cached directories shared between jobs & pipelines per-branch per-runner
3127
cache:
3228
key: $CI_COMMIT_REF_SLUG
@@ -37,8 +33,10 @@ cache:
3733
- ./tmp/Homebrew
3834
# `jest` cache is configured in jest.config.js
3935
- ./tmp/jest/
36+
4037
stages:
4138
- build # Cross-platform library compilation, unit tests
39+
4240
image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
4341
EOF
4442

@@ -48,16 +46,20 @@ printf "\n"
4846
# In the future we can incorporate test durations rather than using
4947
# a static value for the parallel keyword
5048

49+
# Number of parallel shards to split the test suite into
50+
CI_PARALLEL=2
51+
52+
5153
cat << "EOF"
5254
build:linux:
5355
stage: build
5456
needs: []
55-
parallel: 2
57+
parallel: $CI_PARALLEL
5658
script:
5759
- >
5860
nix-shell --run '
59-
npm run build --verbose;
60-
npm test -- --ci --runInBand --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
61+
npm run build --verbose
62+
npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
6163
'
6264
artifacts:
6365
when: always
@@ -67,15 +69,7 @@ build:linux:
6769
coverage_report:
6870
coverage_format: cobertura
6971
path: ./tmp/coverage/cobertura-coverage.xml
70-
paths:
71-
# Only the build:linux preserves the dist
72-
- ./dist
7372
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
74-
rules:
75-
# Runs on staging commits and ignores version commits
76-
- if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
77-
# Runs on tag pipeline where the tag is a prerelease or release version
78-
- if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
7973
EOF
8074

8175
printf "\n"
@@ -84,7 +78,7 @@ cat << "EOF"
8478
build:windows:
8579
stage: build
8680
needs: []
87-
parallel: 2
81+
parallel: $CI_PARALLEL
8882
tags:
8983
- windows
9084
before_script:
@@ -95,17 +89,12 @@ build:windows:
9589
- npm install --ignore-scripts
9690
- $env:Path = "$(npm bin);" + $env:Path
9791
- npm run build --verbose
98-
- npm test -- --ci --runInBand --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
92+
- npm test -- --ci --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
9993
artifacts:
10094
when: always
10195
reports:
10296
junit:
10397
- ./tmp/junit/junit.xml
104-
rules:
105-
# Runs on staging commits and ignores version commits
106-
- if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
107-
# Runs on tag pipeline where the tag is a prerelease or release version
108-
- if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
10998
EOF
11099

111100
printf "\n"
@@ -114,7 +103,7 @@ cat << "EOF"
114103
build:macos:
115104
stage: build
116105
needs: []
117-
parallel: 2
106+
parallel: $CI_PARALLEL
118107
tags:
119108
- shared-macos-amd64
120109
image: macos-11-xcode-12
@@ -130,17 +119,12 @@ build:macos:
130119
- npm install --ignore-scripts
131120
- export PATH="$(npm bin):$PATH"
132121
- npm run build --verbose
133-
- npm test -- --ci --runInBand --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
122+
- npm test -- --ci --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
134123
artifacts:
135124
when: always
136125
reports:
137126
junit:
138127
- ./tmp/junit/junit.xml
139-
rules:
140-
# Runs on staging commits and ignores version commits
141-
- if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
142-
# Runs on tag pipeline where the tag is a prerelease or release version
143-
- if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
144128
EOF
145129

146130
printf "\n"

‎scripts/test-pipelines.sh‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,9 @@ shopt -s nullglob
55

66
# Quote the heredoc to prevent shell expansion
77
cat << "EOF"
8-
workflow:
9-
rules:
10-
# Disable merge request pipelines
11-
- if: $CI_MERGE_REQUEST_ID
12-
when: never
13-
- when: always
148
default:
159
interruptible: true
10+
1611
variables:
1712
GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
1813
GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
@@ -25,6 +20,7 @@ variables:
2520
# It must use an absolute path, otherwise ts-node calls will CWD
2621
TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
2722
TS_CACHED_TRANSPILE_PORTABLE: "true"
23+
2824
# Cached directories shared between jobs & pipelines per-branch per-runner
2925
cache:
3026
key: $CI_COMMIT_REF_SLUG
@@ -33,8 +29,10 @@ cache:
3329
- ./tmp/ts-node-cache/
3430
# `jest` cache is configured in jest.config.js
3531
- ./tmp/jest/
32+
3633
stages:
3734
- check # Linting, unit tests
35+
3836
image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
3937
EOF
4038

@@ -43,16 +41,19 @@ printf "\n"
4341
# Using shards to optimise tests
4442
# In the future we can incorporate test durations rather than using
4543
# a static value for the parallel keyword
44+
45+
# Number of parallel shards to split the test suite into
46+
CI_PARALLEL=2
47+
4648
cat << "EOF"
4749
check:test:
4850
stage: check
4951
needs: []
50-
parallel: 2
52+
parallel: $CI_PARALLEL
5153
script:
5254
- >
5355
nix-shell --run '
54-
npm run build --verbose;
55-
npm test -- --ci --runInBand --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
56+
npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
5657
'
5758
artifacts:
5859
when: always
@@ -63,12 +64,6 @@ check:test:
6364
coverage_format: cobertura
6465
path: ./tmp/coverage/cobertura-coverage.xml
6566
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
66-
rules:
67-
# Runs on staging commits and ignores version commits
68-
- if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
69-
# Manually run on commits other than master and staging and ignore version commits
70-
- if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
71-
when: manual
7267
EOF
7368

7469
printf "\n"

0 commit comments

Comments
 (0)