Skip to content

Commit b50dbb8

Browse files
alalazoalalazo
authored
pipelines: simplify and lint aws-pcluster-* (spack#47989)
1 parent 30c0035 commit b50dbb8

File tree

3 files changed

+59
-163
lines changed

3 files changed

+59
-163
lines changed

‎lib/spack/spack/util/spack_yaml.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,10 +416,12 @@ def dump_config(data, stream, *, default_flow_style=False, blame=False):
416416
if blame:
417417
handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE)
418418
handler.yaml.default_flow_style = default_flow_style
419+
handler.yaml.width = maxint
419420
return _dump_annotated(handler, data, stream)
420421

421422
handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)
422423
handler.yaml.default_flow_style = default_flow_style
424+
handler.yaml.width = maxint
423425
return handler.dump(data, stream)
424426

425427

‎share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml‎

Lines changed: 26 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
spack:
22
view: false
3-
43
definitions:
54
- apps:
65
- gromacs
@@ -9,20 +8,18 @@ spack:
98
- openfoam
109
- quantum-espresso
1110
- wrf
12-
1311
- targets:
14-
- 'target=neoverse_v1'
15-
- 'target=neoverse_n1'
16-
12+
- target=neoverse_v1
13+
- target=neoverse_n1
1714
specs:
1815
- matrix:
19-
- [$apps]
20-
- [$targets]
16+
- [$apps]
17+
- [$targets]
2118
ci:
2219
pipeline-gen:
2320
- build-job:
24-
image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
25-
tags: ["aarch64"]
21+
image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
22+
tags: [aarch64]
2623
before_script:
2724
- - . "./share/spack/setup-env.sh"
2825
- . /etc/profile.d/modules.sh
@@ -31,27 +28,12 @@ spack:
3128
- export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/binutils-2.37-2yxz3xsjfmesxujxtlrgcctxlyilynmp/bin:$PATH
3229
- signing-job:
3330
before_script:
34-
# Do not distribute Intel & ARM binaries
35-
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
36-
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
37-
31+
# Do not distribute Intel & ARM binaries
32+
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
33+
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
3834
cdash:
3935
build-group: AWS Packages
40-
4136
compilers:
42-
- compiler:
43-
environment: {}
44-
extra_rpaths: []
45-
flags: {}
46-
modules: []
47-
operating_system: amzn2
48-
paths:
49-
cc: /usr/bin/gcc
50-
cxx: /usr/bin/g++
51-
f77: /usr/bin/gfortran
52-
fc: /usr/bin/gfortran
53-
spec: gcc@=7.3.1
54-
target: aarch64
5537
- compiler:
5638
environment: {}
5739
extra_rpaths: []
@@ -65,41 +47,34 @@ spack:
6547
fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/gfortran
6648
spec: gcc@=12.4.0
6749
target: aarch64
68-
6950
packages:
7051
acfl:
7152
require:
72-
- one_of: ["%gcc target=aarch64"]
73-
message: "Clang based compilers need GCC libraries and they should be made available for the wide range of CPUs they actually support.
74-
Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
53+
- '%gcc target=aarch64'
7554
gromacs:
7655
require:
77-
- one_of:
78-
- "[email protected] %arm ^fftw ^openmpi"
79-
- "[email protected] %gcc ^armpl-gcc ^openmpi"
56+
- [email protected] %gcc ^armpl-gcc ^openmpi
8057
libfabric:
8158
buildable: true
8259
externals:
83-
- prefix: /opt/amazon/efa/
84-
spec: [email protected]
85-
require: ['fabrics=shm,efa']
60+
- prefix: /opt/amazon/efa/
61+
spec: [email protected]
62+
require:
63+
- fabrics=shm,efa
8664
llvm:
8765
variants: ~lldb
8866
mpas-model:
8967
require:
90-
- one_of:
91-
- "precision=single make_target=llvm %arm ^parallelio+pnetcdf"
92-
- "precision=single %gcc ^parallelio+pnetcdf"
68+
- precision=single %gcc ^parallelio+pnetcdf
9369
mpich:
94-
require: "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm"
70+
require:
71+
- mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
9572
nvhpc:
9673
require:
97-
- one_of:
98-
- "nvhpc %gcc target=aarch64"
99-
message: "NVHPC should be built with GCC and should be made available for the wide range of CPUs they actually support.
100-
Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
74+
- nvhpc %gcc target=aarch64
10175
openfoam:
102-
require: "openfoam %gcc ^[email protected]"
76+
require:
77+
- openfoam ^[email protected]
10378
openmpi:
10479
variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
10580
require: '@4:'
@@ -108,19 +83,15 @@ spack:
10883
# require:
10984
# - one_of: ["palace cxxflags=\"-include cstdint\" ^[email protected]"]
11085
pmix:
111-
require: "pmix@3:"
86+
require: 'pmix@3:'
11287
quantum-espresso:
113-
require: "[email protected] %gcc ^armpl-gcc"
88+
require:
89+
- [email protected] %gcc ^armpl-gcc
11490
slurm:
11591
buildable: false
11692
externals:
117-
- prefix: /opt/slurm
118-
spec: [email protected] +pmix
119-
wrf:
120-
require:
121-
- one_of:
122-
- "wrf%arm"
123-
- "wrf%gcc"
93+
- prefix: /opt/slurm
94+
spec: [email protected] +pmix
12495
all:
12596
compiler: [gcc, arm, nvhpc, clang]
12697
providers:

‎share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml‎

Lines changed: 31 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,28 @@
11
spack:
22
view: false
3-
43
definitions:
54
- apps:
65
- gromacs %oneapi
76
- lammps %oneapi
8-
# earliest oneapi version with fix does not run on AmazonLinux2, see https://github.com/spack/spack/pull/46457
9-
# - mpas-model %oneapi
7+
# earliest oneapi version with fix does not run on AmazonLinux2, see https://github.com/spack/spack/pull/46457
8+
# - mpas-model %oneapi
109
- openfoam %gcc
11-
- palace %oneapi ^superlu-dist%oneapi # hack: force fortran-rt provider through superlu-dist
12-
# TODO: Find out how to make +ipo cmake flag work.
13-
# - quantum-espresso %oneapi
10+
- palace %oneapi ^superlu-dist%oneapi # hack: force fortran-rt provider through superlu-dist
11+
# TODO: Find out how to make +ipo cmake flag work.
12+
# - quantum-espresso %oneapi
1413
- openmpi %oneapi
1514
- wrf %oneapi
16-
1715
- targets:
18-
- 'target=x86_64_v4'
19-
- 'target=x86_64_v3'
20-
16+
- target=x86_64_v4
17+
- target=x86_64_v3
2118
specs:
2219
- matrix:
23-
- [$apps]
24-
- [$targets]
20+
- [$apps]
21+
- [$targets]
2522
ci:
2623
pipeline-gen:
2724
- build-job:
28-
image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
25+
image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
2926
before_script:
3027
- - . "./share/spack/setup-env.sh"
3128
- . /etc/profile.d/modules.sh
@@ -34,13 +31,11 @@ spack:
3431
- export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/binutils-2.37-qvccg7zpskturysmr4bzbsfrx34kvazo/bin:$PATH
3532
- signing-job:
3633
before_script:
37-
# Do not distribute Intel & ARM binaries
38-
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
39-
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
40-
34+
# Do not distribute Intel & ARM binaries
35+
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
36+
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
4137
cdash:
4238
build-group: AWS Packages
43-
4439
compilers:
4540
- compiler:
4641
environment: {}
@@ -69,131 +64,59 @@ spack:
6964
fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/ifx
7065
spec: oneapi@=2024.1.0
7166
target: x86_64
72-
7367
packages:
74-
cpio:
75-
require:
76-
- one_of:
77-
- "cflags=-std=c18 target=x86_64_v4"
78-
- "cflags=-std=c18 target=x86_64_v3"
79-
when: "%intel"
8068
gettext:
8169
# Newer gettext cannot build with gcc@12 and old AL2 glibc headers
8270
# Older gettext versions do not build correctly with oneapi.
8371
require:
84-
- one_of:
85-
- '@:0.20'
86-
- '%oneapi'
72+
- one_of: ['@:0.20 %gcc', '%oneapi']
8773
gromacs:
8874
require:
89-
- one_of:
90-
- "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v4"
91-
- "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v3"
92-
when: "%intel"
93-
- one_of:
94-
- "+intel_provided_gcc target=x86_64_v4 ^intel-oneapi-mkl"
95-
- "+intel_provided_gcc target=x86_64_v3 ^intel-oneapi-mkl"
96-
when: "%oneapi"
97-
intel-oneapi-compilers:
98-
require: "intel-oneapi-compilers %gcc target=x86_64_v3"
75+
- +intel_provided_gcc ^intel-oneapi-mkl
9976
intel-oneapi-mpi:
10077
variants: +external-libfabric generic-names=True
10178
lammps:
10279
require:
103-
- one_of:
104-
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
105-
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
106-
when: "%intel"
107-
- one_of:
108-
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
109-
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
110-
when: "%oneapi"
111-
libidn2:
112-
require:
113-
- one_of:
114-
- "cflags=-std=c18 target=x86_64_v4"
115-
- "cflags=-std=c18 target=x86_64_v3"
116-
when: "%intel"
80+
- lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl
81+
- one_of: [+intel target=x86_64_v4, target=x86_64_v3]
11782
libfabric:
11883
buildable: true
11984
externals:
120-
- prefix: /opt/amazon/efa/
121-
spec: [email protected]
122-
require: ['fabrics=shm,efa']
123-
libunistring:
85+
- prefix: /opt/amazon/efa/
86+
spec: [email protected]
12487
require:
125-
- one_of:
126-
- "cflags=-std=c18 target=x86_64_v4"
127-
- "cflags=-std=c18 target=x86_64_v3"
128-
when: "%intel"
88+
- fabrics=shm,efa
12989
mpas-model:
13090
require:
131-
- one_of:
132-
- "precision=single ^parallelio+pnetcdf target=x86_64_v4"
133-
- "precision=single ^parallelio+pnetcdf target=x86_64_v3"
134-
when: "%intel"
135-
- one_of:
136-
- "precision=single ^parallelio+pnetcdf target=x86_64_v4"
137-
- "precision=single ^parallelio+pnetcdf target=x86_64_v3"
138-
when: "%oneapi"
91+
- spec: precision=single ^parallelio+pnetcdf
92+
when: '%oneapi'
13993
mpich:
14094
require:
141-
- one_of:
142-
- "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v4"
143-
- "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v3"
95+
- mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
14496
openfoam:
14597
require:
146-
- one_of:
147-
- "openfoam %gcc ^[email protected] target=x86_64_v4"
148-
- "openfoam %gcc ^[email protected] target=x86_64_v3"
98+
- openfoam %gcc ^[email protected]
14999
openmpi:
150100
variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
151101
require:
152-
- one_of:
153-
- "openmpi @4: target=x86_64_v4"
154-
- "openmpi @4: target=x86_64_v3"
102+
- 'openmpi @4:'
155103
palace:
156104
require:
157-
- one_of:
158-
- "palace ^[email protected] target=x86_64_v4"
159-
- "palace ^[email protected] target=x86_64_v3"
160-
when: "%oneapi"
161-
- one_of:
162-
- "palace ^[email protected]"
163-
when: "%gcc"
105+
- palace ^[email protected]
164106
pmix:
165107
require:
166-
- one_of:
167-
- "pmix@3: target=x86_64_v4"
168-
- "pmix@3: target=x86_64_v3"
108+
- 'pmix@3:'
169109
quantum-espresso:
170110
require:
171-
- one_of:
172-
- "[email protected] ^intel-oneapi-mkl+cluster target=x86_64_v4"
173-
- "[email protected] ^intel-oneapi-mkl+cluster target=x86_64_v3"
174-
when: "%intel"
175-
- one_of:
176-
- "[email protected] ^intel-oneapi-mkl+cluster target=x86_64_v4"
177-
- "[email protected] ^intel-oneapi-mkl+cluster target=x86_64_v3"
178-
when: "%oneapi"
111+
- [email protected] ^intel-oneapi-mkl+cluster
179112
slurm:
180113
buildable: false
181114
externals:
182-
- prefix: /opt/slurm
183-
spec: [email protected] +pmix
115+
- prefix: /opt/slurm
116+
spec: [email protected] +pmix
184117
wrf:
185118
require:
186-
- one_of:
187-
- "wrf@4 build_type=dm+sm target=x86_64_v4"
188-
- "wrf@4 build_type=dm+sm target=x86_64_v3"
189-
- "[email protected] +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
190-
when: "%intel"
191-
- one_of:
192-
- "wrf@4 build_type=dm+sm target=x86_64_v4"
193-
- "wrf@4 build_type=dm+sm target=x86_64_v3"
194-
- "[email protected] +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
195-
when: "%oneapi"
196-
119+
- wrf@4 build_type=dm+sm
197120
all:
198121
compiler: [oneapi, gcc]
199122
permissions:

0 commit comments

Comments
 (0)