Skip to content

Commit e67e039

Browse files
authored
ci: Limit number of parallel jobs in "upload-conan-deps" (#5781)
- This should prevent Artifactory from being overloaded by too many requests at a time. - Uses "max-parallel" to limit the build job to 10 simultaneous instances. - Only run the minimal matrix on PRs.
1 parent 148f669 commit e67e039

File tree

4 files changed

+74
-105
lines changed

4 files changed

+74
-105
lines changed

.github/scripts/strategy-matrix/generate.py

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,17 @@
22
import argparse
33
import itertools
44
import json
5-
import re
5+
from pathlib import Path
6+
from dataclasses import dataclass
7+
8+
THIS_DIR = Path(__file__).parent.resolve()
9+
10+
@dataclass
11+
class Config:
12+
architecture: list[dict]
13+
os: list[dict]
14+
build_type: list[str]
15+
cmake_args: list[str]
616

717
'''
818
Generate a strategy matrix for GitHub Actions CI.
@@ -18,9 +28,9 @@
1828
- Certain Debian Bookworm configurations will change the reference fee, enable
1929
codecov, and enable voidstar in PRs.
2030
'''
21-
def generate_strategy_matrix(all: bool, architecture: list[dict], os: list[dict], build_type: list[str], cmake_args: list[str]) -> dict:
31+
def generate_strategy_matrix(all: bool, config: Config) -> list:
2232
configurations = []
23-
for architecture, os, build_type, cmake_args in itertools.product(architecture, os, build_type, cmake_args):
33+
for architecture, os, build_type, cmake_args in itertools.product(config.architecture, config.os, config.build_type, config.cmake_args):
2434
# The default CMake target is 'all' for Linux and MacOS and 'install'
2535
# for Windows, but it can get overridden for certain configurations.
2636
cmake_target = 'install' if os["distro_name"] == 'windows' else 'all'
@@ -158,21 +168,30 @@ def generate_strategy_matrix(all: bool, architecture: list[dict], os: list[dict]
158168
'architecture': architecture,
159169
})
160170

161-
return {'include': configurations}
171+
return configurations
172+
173+
174+
def read_config(file: Path) -> Config:
175+
config = json.loads(file.read_text())
176+
if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None:
177+
raise Exception('Invalid configuration file.')
178+
179+
return Config(**config)
162180

163181

164182
if __name__ == '__main__':
165183
parser = argparse.ArgumentParser()
166184
parser.add_argument('-a', '--all', help='Set to generate all configurations (generally used when merging a PR) or leave unset to generate a subset of configurations (generally used when committing to a PR).', action="store_true")
167-
parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=True, type=str)
185+
parser.add_argument('-c', '--config', help='Path to the JSON file containing the strategy matrix configurations.', required=False, type=Path)
168186
args = parser.parse_args()
169187

170-
# Load the JSON configuration file.
171-
config = None
172-
with open(args.config, 'r') as f:
173-
config = json.load(f)
174-
if config['architecture'] is None or config['os'] is None or config['build_type'] is None or config['cmake_args'] is None:
175-
raise Exception('Invalid configuration file.')
188+
matrix = []
189+
if args.config is None or args.config == '':
190+
matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "linux.json"))
191+
matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "macos.json"))
192+
matrix += generate_strategy_matrix(args.all, read_config(THIS_DIR / "windows.json"))
193+
else:
194+
matrix += generate_strategy_matrix(args.all, read_config(args.config))
176195

177196
# Generate the strategy matrix.
178-
print(f'matrix={json.dumps(generate_strategy_matrix(args.all, config['architecture'], config['os'], config['build_type'], config['cmake_args']))}')
197+
print(f'matrix={json.dumps({"include": matrix})}')

.github/workflows/reusable-strategy-matrix.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ on:
55
inputs:
66
os:
77
description: 'The operating system to use for the build ("linux", "macos", "windows").'
8-
required: true
8+
required: false
99
type: string
1010
strategy_matrix:
1111
# TODO: Support additional strategies, e.g. "ubuntu" for generating all Ubuntu configurations.
@@ -35,4 +35,4 @@ jobs:
3535
- name: Generate strategy matrix
3636
working-directory: .github/scripts/strategy-matrix
3737
id: generate
38-
run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} --config=${{ inputs.os }}.json >> "${GITHUB_OUTPUT}"
38+
run: ./generate.py ${{ inputs.strategy_matrix == 'all' && '--all' || '' }} ${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }} >> "${GITHUB_OUTPUT}"

.github/workflows/reusable-upload-conan-deps-os.yml

Lines changed: 0 additions & 78 deletions
This file was deleted.

.github/workflows/upload-conan-deps.yml

Lines changed: 41 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ on:
2121
- .github/workflows/upload-conan-deps.yml
2222

2323
- .github/workflows/reusable-strategy-matrix.yml
24-
- .github/workflows/reusable-upload-conan-deps-os.yml
2524

2625
- .github/actions/build-deps/action.yml
2726
- ".github/scripts/strategy-matrix/**"
@@ -34,7 +33,6 @@ on:
3433
- .github/workflows/upload-conan-deps.yml
3534

3635
- .github/workflows/reusable-strategy-matrix.yml
37-
- .github/workflows/reusable-upload-conan-deps-os.yml
3836

3937
- .github/actions/build-deps/action.yml
4038
- ".github/scripts/strategy-matrix/**"
@@ -47,16 +45,46 @@ concurrency:
4745
cancel-in-progress: true
4846

4947
jobs:
48+
generate-matrix:
49+
uses: ./.github/workflows/reusable-strategy-matrix.yml
50+
with:
51+
strategy_matrix: ${{ github.event_name == 'pull_request' && 'minimal' || 'all' }}
52+
5053
run-upload-conan-deps:
54+
needs:
55+
- generate-matrix
5156
strategy:
52-
fail-fast: true
53-
matrix:
54-
os: ["linux", "macos", "windows"]
55-
uses: ./.github/workflows/reusable-upload-conan-deps-os.yml
56-
with:
57-
force_source_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }}
58-
force_upload: ${{ github.event.inputs.force_upload == 'true' }}
59-
os: ${{ matrix.os }}
60-
secrets:
61-
CONAN_USERNAME: ${{ secrets.CONAN_REMOTE_USERNAME }}
62-
CONAN_PASSWORD: ${{ secrets.CONAN_REMOTE_PASSWORD }}
57+
fail-fast: false
58+
matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
59+
max-parallel: 10
60+
runs-on: ${{ matrix.architecture.runner }}
61+
container: ${{ contains(matrix.architecture.platform, 'linux') && format('ghcr.io/xrplf/ci/{0}-{1}:{2}-{3}', matrix.os.distro_name, matrix.os.distro_version, matrix.os.compiler_name, matrix.os.compiler_version) || null }}
62+
63+
steps:
64+
- name: Cleanup workspace
65+
if: ${{ runner.os == 'macOS' }}
66+
uses: XRPLF/actions/.github/actions/cleanup-workspace@3f044c7478548e3c32ff68980eeb36ece02b364e
67+
68+
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
69+
- name: Prepare runner
70+
uses: XRPLF/actions/.github/actions/prepare-runner@638e0dc11ea230f91bd26622fb542116bb5254d5
71+
with:
72+
disable_ccache: false
73+
74+
- name: Setup Conan
75+
uses: ./.github/actions/setup-conan
76+
77+
- name: Build dependencies
78+
uses: ./.github/actions/build-deps
79+
with:
80+
build_dir: .build
81+
build_type: ${{ matrix.build_type }}
82+
force_build: ${{ github.event_name == 'schedule' || github.event.inputs.force_source_build == 'true' }}
83+
84+
- name: Login to Conan
85+
if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request'
86+
run: conan remote login -p ${{ secrets.CONAN_PASSWORD }} ${{ inputs.conan_remote_name }} ${{ secrets.CONAN_USERNAME }}
87+
88+
- name: Upload Conan packages
89+
if: github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' && github.event_name != 'schedule'
90+
run: conan upload "*" -r=${{ inputs.conan_remote_name }} --confirm ${{ github.event.inputs.force_upload == 'true' && '--force' || '' }}

0 commit comments

Comments
 (0)