Skip to content

Commit afc9b5f

Browse files
puehringerpuehringer
authored
CI speed improvements by running things in parallel (#76)
* Update build-node-python.yml * Add run_parallel and run_install * Fix run_parallel check * Fix parallel command * Parallelize even more * Fix parallelize * Add env * Parallelize even further * Fix bug * Fix bug * Run everything in parallel * Correct order of decrypt * Many changes * Add $JOBS flag * Fix brackets * Make $JOBS parameters * Add comment * Fix conditions * Make parallel jobs an array * Add webpack * Rename $JOBS to $N_JOBS * Update build-node-python.yml * Update build-node-python.yml * Update build-node-python.yml * Remove build-node and build-python actions * Keep build node and python as single, separate jobs * Fix concurrency group * Run only enabled jobs * Update build-node-python.yml
1 parent b47f340 commit afc9b5f

File tree

10 files changed

+334
-243
lines changed

10 files changed

+334
-243
lines changed

‎.github/actions/build-node-python/action.yml‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
name: build-node-python
2+
description: Build node sources via yarn and python via make
3+
author: datavisyn
4+
5+
inputs:
6+
github_ro_token:
7+
description: "github read-only token"
8+
default: "admin"
9+
required: true
10+
run_parallel:
11+
description: "enables parallel running of node and python tasks"
12+
default: true
13+
required: true
14+
# Node
15+
enable_node:
16+
description: "enables the node part of the action"
17+
default: true
18+
required: true
19+
node_version:
20+
description: "node version to use"
21+
default: "20.9"
22+
required: true
23+
npm_registry:
24+
description: "npm registry"
25+
default: "https://registry.npmjs.org/"
26+
required: true
27+
enable_node_cache:
28+
description: "enables the yarn cache download and upload"
29+
required: false
30+
default: true
31+
download_yarn_lock:
32+
description: "downloads the node-yarn-lock artifact (only available when build-node was called before)"
33+
required: false
34+
default: false
35+
upload_yarn_lock:
36+
description: "uploads the node-yarn-lock artifact"
37+
required: false
38+
default: true
39+
run_node_lint:
40+
default: true
41+
required: false
42+
run_playwright_browser_install:
43+
default: false
44+
required: false
45+
run_node_test:
46+
default: true
47+
required: false
48+
run_node_build:
49+
default: true
50+
required: false
51+
run_node_bundle:
52+
default: false
53+
required: false
54+
chromatic_enable:
55+
description: "Enable Chromatic tests"
56+
required: false
57+
type: boolean
58+
default: false
59+
chromatic_project_token:
60+
description: "Chromatic project token"
61+
required: false
62+
# Python
63+
enable_python:
64+
description: "enables the python part of the action"
65+
default: true
66+
required: true
67+
python_version:
68+
description: "python version to use"
69+
default: "3.10"
70+
required: true
71+
enable_python_cache:
72+
description: "enables the pip cache download and upload"
73+
required: false
74+
default: true
75+
run_python_lint:
76+
default: true
77+
required: false
78+
run_python_test:
79+
default: true
80+
required: false
81+
run_python_build:
82+
default: true
83+
required: false
84+
85+
runs:
86+
using: "composite"
87+
steps:
88+
# Node
89+
- name: Set up node version
90+
uses: actions/setup-node@v3
91+
if: inputs.enable_node == 'true'
92+
with:
93+
node-version: ${{ inputs.node_version }}
94+
registry-url: ${{ inputs.npm_registry }}
95+
- name: Show node and npm version
96+
if: inputs.enable_node == 'true'
97+
run: |
98+
corepack enable
99+
node -v
100+
npm -v
101+
shell: bash
102+
# Python
103+
- name: Set up python
104+
uses: actions/setup-python@v5
105+
if: inputs.enable_python == 'true'
106+
with:
107+
python-version: ${{ inputs.python_version }}
108+
cache: ${{ inputs.enable_python_cache == 'true' && 'pip' || null }}
109+
- name: Install additional python requirements
110+
if: inputs.enable_python == 'true'
111+
run: |
112+
pip install setuptools wheel
113+
python --version
114+
pip --version
115+
shell: bash
116+
# General
117+
- name: Git config
118+
if: inputs.github_ro_token != ''
119+
run: |
120+
if [ -f ~/.gitconfig ]; then
121+
rm ~/.gitconfig
122+
touch ~/.gitconfig
123+
fi
124+
git config --global --replace-all url."https://[email protected]/".insteadOf ssh://[email protected]/
125+
git config --add --global url."https://$GITHUB_TOKEN@github".insteadOf https://github
126+
git config --add --global url."https://[email protected]/".insteadOf [email protected]:
127+
env:
128+
GITHUB_TOKEN: ${{ inputs.github_ro_token }}
129+
shell: bash
130+
- uses: actions/download-artifact@v3
131+
# TODO: Remove the variable whenever we can ensure that build-node was called before
132+
if: inputs.enable_node == 'true' && inputs.download_yarn_lock == 'true'
133+
with:
134+
name: node-yarn-lock
135+
# Enable yarn download cache, @see https://github.com/actions/cache/tree/main/save#always-save-cache and https://github.com/actions/setup-node/issues/325
136+
- name: Restore yarn cache
137+
uses: actions/cache/restore@v3
138+
if: inputs.enable_node == 'true' && inputs.enable_node_cache == 'true'
139+
with:
140+
# This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder
141+
path: ~/.yarn/berry/cache/
142+
key: yarn-download-cache-${{ hashFiles('package.json') }}
143+
restore-keys: |
144+
yarn-download-cache-
145+
# Parallel build
146+
- name: Build node and python in parallel
147+
id: build
148+
run: |
149+
# Run node and python in parallel
150+
151+
# Define the node sequence of commands
152+
node_job() {
153+
set -e
154+
yarn install --no-immutable --inline-builds
155+
156+
parallel_jobs=()
157+
if [ "$RUN_NODE_LINT" = "true" ]; then
158+
parallel_jobs+=("yarn run lint")
159+
fi
160+
if [ "$RUN_NODE_TEST" = "true" ]; then
161+
parallel_jobs+=("yarn run test")
162+
fi
163+
if [ "$RUN_NODE_BUILD" = "true" ]; then
164+
parallel_jobs+=("yarn run build")
165+
fi
166+
if [ "$RUN_NODE_BUNDLE" = "true" ]; then
167+
parallel_jobs+=("yarn run bundle:prod || yarn run webpack:prod")
168+
fi
169+
if [ "$RUN_PLAYWRIGHT_BROWSER_INSTALL" = "true" ]; then
170+
parallel_jobs+=("yarn playwright install --with-deps chromium")
171+
fi
172+
173+
parallel --jobs $1 --lb --halt-on-error 2 --verbose ::: "${parallel_jobs[@]}"
174+
}
175+
176+
# Define the python sequence of commands
177+
python_job() {
178+
set -e
179+
make develop
180+
181+
parallel_jobs=()
182+
if [ "$RUN_PYTHON_LINT" = "true" ]; then
183+
parallel_jobs+=("make lint check-format")
184+
fi
185+
if [ "$RUN_PYTHON_TEST" = "true" ]; then
186+
parallel_jobs+=("make test")
187+
fi
188+
if [ "$RUN_PYTHON_BUILD" = "true" ]; then
189+
parallel_jobs+=("make build")
190+
fi
191+
192+
parallel --jobs $1 --lb --halt-on-error 2 --verbose ::: "${parallel_jobs[@]}"
193+
}
194+
195+
# Export the functions so they can be used by GNU parallel
196+
export -f node_job
197+
export -f python_job
198+
199+
# If RUN_PARALLEL is set, set --jobs to 0, otherwise to 1
200+
N_JOBS=1
201+
if [ "$RUN_PARALLEL" = "true" ]; then
202+
N_JOBS=0
203+
fi
204+
205+
final_jobs=()
206+
if [ "$RUN_NODE" = "true" ]; then
207+
final_jobs+=("node_job $N_JOBS")
208+
fi
209+
if [ "$RUN_PYTHON" = "true" ]; then
210+
final_jobs+=("python_job $N_JOBS")
211+
fi
212+
213+
parallel --jobs $N_JOBS --lb --halt-on-error 2 --verbose ::: "${final_jobs[@]}"
214+
shell: bash
215+
env:
216+
RUN_PARALLEL: ${{ inputs.run_parallel }}
217+
RUN_NODE: ${{ inputs.enable_node }}
218+
RUN_PYTHON: ${{ inputs.enable_python }}
219+
RUN_PLAYWRIGHT_BROWSER_INSTALL: ${{ inputs.run_playwright_browser_install }}
220+
RUN_NODE_LINT: ${{ inputs.run_node_lint }}
221+
RUN_NODE_TEST: ${{ inputs.run_node_test }}
222+
RUN_NODE_BUILD: ${{ inputs.run_node_build }}
223+
RUN_NODE_BUNDLE: ${{ inputs.run_node_bundle }}
224+
RUN_PYTHON_LINT: ${{ inputs.run_python_lint }}
225+
RUN_PYTHON_TEST: ${{ inputs.run_python_test }}
226+
RUN_PYTHON_BUILD: ${{ inputs.run_python_build }}
227+
# Node
228+
- name: Save yarn cache
229+
uses: actions/cache/save@v3
230+
if: inputs.enable_node == 'true' && steps.build.outcome == 'success' && inputs.enable_node_cache == 'true'
231+
with:
232+
# This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder
233+
path: ~/.yarn/berry/cache/
234+
key: yarn-download-cache-${{ hashFiles('package.json') }}
235+
- uses: actions/upload-artifact@v3
236+
if: inputs.enable_node == 'true' && inputs.upload_yarn_lock == 'true'
237+
with:
238+
name: node-yarn-lock
239+
path: ./yarn.lock
240+
- uses: actions/upload-artifact@v3
241+
if: inputs.enable_node == 'true' && inputs.run_node_bundle == 'true'
242+
with:
243+
name: node-bundle
244+
path: ./bundles
245+
# Chromatic
246+
- name: Run Chromatic
247+
if: inputs.enable_node == 'true' && inputs.chromatic_enable == 'true'
248+
uses: chromaui/[email protected]
249+
with:
250+
projectToken: ${{ inputs.chromatic_project_token }}

‎.github/actions/build-node/action.yml‎

Lines changed: 0 additions & 123 deletions
This file was deleted.

0 commit comments

Comments
 (0)