Skip to content

Commit 2e08c74

Browse files
ursgursg
authored
Merge pull request #621 from fmihpc/dev
Merge dev into master for Vlasiator 5.2 release.
2 parents 82a5bdf + 32a49cc commit 2e08c74

File tree

147 files changed

+15694
-3777
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

147 files changed

+15694
-3777
lines changed

‎MAKE/Makefile.carrington_gcc_openmpi‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Makefile for the Carrington Cluster at UH.
2+
#
3+
# Recommended setup:
4+
# * Compile on an interactive node with the below modules;
5+
# See the It4Science wiki or the Vorna guide
6+
# https://github.com/fmihpc/analysator/wiki/Vorna-guide
7+
# for the current best practice.
8+
# * Use mpirun to run (no need for the long command below it seems)
9+
# * (this was used previously, kept for documentation purposes) mpirun -mca pml ucx --mca btl ^vader,tcp,openib -x UCX_NET_DEVICES=mlx5_0:1 -x UCX_TLS=rc,sm -x UCX_IB_ADDR_TYPE=ib_global $executable --run_config $configfile
10+
#
11+
# Required modules:
12+
#
13+
# module load gnu9/9.4.0
14+
# module load openmpi4/4.1.1
15+
# module load pmix/3.1.4
16+
# Oneliner for command line:
17+
# module purge ; module load gnu9/9.4.0 ; module load openmpi4/4.1.1 ; module load pmix/3.1.4
18+
#
19+
# NOTE in the job script, you have to load
20+
#
21+
# module purge
22+
# module load GCC/10.2.0
23+
# module load OpenMPI/4.0.5-GCC-10.2.0
24+
#
25+
# otherwise the VLSV output is broken.
26+
#
27+
28+
CMP = mpicxx
29+
LNK = mpicxx
30+
31+
#======== Vectorization ==========
32+
#Set vector backend type for vlasov solvers, sets precision and length.
33+
#Options:
34+
# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
35+
# AVX512: VEC8D_AGNER, VEC16F_AGNER
36+
# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
37+
38+
ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
39+
#Single-precision
40+
VECTORCLASS = VEC8F_AGNER
41+
else
42+
#Double-precision
43+
VECTORCLASS = VEC4D_AGNER
44+
endif
45+
46+
#======= Compiler and compilation flags =========
47+
# NOTES on compiler flags:
48+
# CXXFLAGS is for compiler flags, they are always used
49+
# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
50+
# LDFLAGS flags for linker
51+
52+
#-DNO_WRITE_AT_ALL: Define to disable write at all to
53+
# avoid memleak (much slower IO)
54+
#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition
55+
# errors that come up when using
56+
# mpi.h in c++ on Cray
57+
#
58+
# CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK
59+
60+
FLAGS =
61+
62+
#GNU flags:
63+
CC_BRAND = gcc
64+
CC_BRAND_VERSION = 9.4.0
65+
CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx -march=znver2 #-flto
66+
testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -mavx -march=znver2
67+
68+
MATHFLAGS = -ffast-math
69+
LDFLAGS = -lrt -lgfortran -std=c++17 -lgomp
70+
LIB_MPI = -lgomp -lmpi
71+
72+
#======== PAPI ==========
73+
#Add PAPI_MEM define to use papi to report memory consumption?
74+
CXXFLAGS += -DPAPI_MEM
75+
testpackage: CXXFLAGS += -DPAPI_MEM
76+
77+
#======== Allocator =========
78+
#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
79+
#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
80+
CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
81+
testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
82+
83+
84+
# BOOST_VERSION = current trilinos version
85+
# ZOLTAN_VERSION = current trilinos verson
86+
#
87+
#======== Libraries ===========
88+
89+
MPT_BRAND = OpenMPI
90+
MPT_VERSION = 4.1.1
91+
92+
JEMALLOC_VERSION = 5.2.1
93+
LIBRARY_PREFIX = /proj/group/spacephysics/libraries
94+
95+
#compiled libraries mostly in modules
96+
LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/phiprof/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/phiprof/lib
97+
INC_PROFILE = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/phiprof/include
98+
99+
LIB_VLSV = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/vlsv
100+
INC_VLSV = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/vlsv
101+
102+
LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/jemalloc/lib -ljemalloc -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/jemalloc/lib
103+
INC_JEMALLOC = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/jemalloc/include
104+
105+
LIB_BOOST = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/boost/lib -lboost_program_options -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/boost/lib
106+
INC_BOOST = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/boost/include
107+
108+
LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/zoltan/lib -lzoltan -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/zoltan/lib
109+
INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/zoltan/include
110+
111+
LIB_PAPI = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/lib
112+
INC_PAPI = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/include
113+
114+
115+
#header libraries
116+
INC_EIGEN = -I$(LIBRARY_PREFIX)/ -I$(LIBRARY_PREFIX)/Eigen/
117+
INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/
118+
INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/
119+
INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass/
120+
121+
122+
123+

‎MAKE/Makefile.hawk_gcc_mpt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ FLAGS =
4343

4444
#GNU flags:
4545
CC_BRAND = gcc
46-
CC_BRAND_VERSION = 9.2.0
47-
CXXFLAGS += -g -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -Wall -Wpedantic -mfma -march=native -mavx2
46+
CC_BRAND_VERSION = 9.2.0_bis
47+
CXXFLAGS += -g -fopenmp -O3 -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -Wall -Wpedantic -mfma -march=native -mavx2
4848
#CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx
4949
not_parallel_tools: CXXFLAGS += -march=native -mno-avx2 -mavx
5050
testpackage: CXXFLAGS = -g -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0
@@ -88,6 +88,8 @@ LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_
8888

8989
INC_VLSV = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv
9090
LIB_VLSV = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv
91+
#INC_VLSV = -I/zhome/academic/HLRS/pri/ipryakem/vlsv
92+
#LIB_VLSV = -L/zhome/academic/HLRS/pri/ipryakem/vlsv -lvlsv
9193

9294
LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof
9395
INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include

‎MAKE/Makefile.mahti_gcc‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ CMP = mpic++
22
LNK = mpic++
33

44
# Modules loaded
5-
# module load gcc boost jemalloc papi openmpi zoltan
5+
# module load gcc boost jemalloc papi openmpi
66

77
#======== Vectorization ==========
88
#Set vector backend type for vlasov solvers, sets precision and length.
@@ -58,28 +58,31 @@ testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
5858

5959
MPT_VERSION = 4.0.3
6060
JEMALLOC_VERSION = 5.2.1
61-
LIBRARY_PREFIX = /users/kempf/libraries
61+
LIBRARY_PREFIX = /users/kempf/libraries_rhel8_openmpi/
6262

6363

6464
#compiled libraries
6565
LIB_BOOST = -lboost_program_options
6666

67-
LIB_ZOLTAN = -lzoltan
67+
INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/include
68+
LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan -Wl,-rpath=$(LIBRARY_PREFIX)/zoltan/lib
69+
6870

6971
LIB_JEMALLOC = -ljemalloc
7072

7173
LIB_PAPI = -lpapi
7274

73-
INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv
74-
LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv
75+
INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv_multiread
76+
LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv_multiread -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv_multiread
77+
7578

7679
LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib
7780
INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include
7881

7982

8083
#header libraries
8184

82-
INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/
83-
INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/
84-
INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/
85-
INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass/
85+
INC_FSGRID = -I$(LIBRARY_PREFIX)../libraries/fsgrid/
86+
INC_EIGEN = -I$(LIBRARY_PREFIX)/../libraries/eigen/
87+
INC_DCCRG = -I$(LIBRARY_PREFIX)/../libraries/dccrg/
88+
INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/../libraries/vectorclass/

‎MAKE/Makefile.ukko_gcc_openmpi‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Makefile for the ukko Cluster at UH.
2+
#
3+
# Recommended setup:
4+
# * Compile on an interactive node with the below modules;
5+
# See the It4Science wiki or the Vorna guide
6+
# https://github.com/fmihpc/analysator/wiki/Vorna-guide
7+
# for the current best practice.
8+
# * Use mpirun to run (no need for the long command below it seems)
9+
# * (this was used previously, kept for documentation purposes) mpirun -mca pml ucx --mca btl ^vader,tcp,openib -x UCX_NET_DEVICES=mlx5_0:1 -x UCX_TLS=rc,sm -x UCX_IB_ADDR_TYPE=ib_global $executable --run_config $configfile
10+
#
11+
# Required modules:
12+
#
13+
# module load gnu9/9.4.0
14+
# module load openmpi4/4.1.1
15+
# module load pmix/3.1.4
16+
# Oneliner for command line:
17+
# module purge ; module load gnu9/9.4.0 ; module load openmpi4/4.1.1 ; module load pmix/3.1.4
18+
#
19+
# NOTE in the job script, you have to load
20+
#
21+
# module purge
22+
# module load GCC/10.2.0
23+
# module load OpenMPI/4.0.5-GCC-10.2.0
24+
#
25+
# otherwise the VLSV output is broken.
26+
#
27+
28+
CMP = mpicxx
29+
LNK = mpicxx
30+
31+
#======== Vectorization ==========
32+
#Set vector backend type for vlasov solvers, sets precision and length.
33+
#Options:
34+
# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
35+
# AVX512: VEC8D_AGNER, VEC16F_AGNER
36+
# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
37+
38+
ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
39+
#Single-precision
40+
VECTORCLASS = VEC8F_AGNER
41+
else
42+
#Double-precision
43+
VECTORCLASS = VEC4D_AGNER
44+
endif
45+
46+
#======= Compiler and compilation flags =========
47+
# NOTES on compiler flags:
48+
# CXXFLAGS is for compiler flags, they are always used
49+
# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
50+
# LDFLAGS flags for linker
51+
52+
#-DNO_WRITE_AT_ALL: Define to disable write at all to
53+
# avoid memleak (much slower IO)
54+
#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition
55+
# errors that come up when using
56+
# mpi.h in c++ on Cray
57+
#
58+
# CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK
59+
60+
FLAGS =
61+
62+
#GNU flags:
63+
CC_BRAND = gcc
64+
CC_BRAND_VERSION = 9.4.0
65+
CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx -march=znver2 #-flto
66+
testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -mavx -march=znver2
67+
68+
MATHFLAGS = -ffast-math
69+
LDFLAGS = -lrt -lgfortran -std=c++17 -lgomp
70+
LIB_MPI = -lgomp -lmpi
71+
72+
#======== PAPI ==========
73+
#Add PAPI_MEM define to use papi to report memory consumption?
74+
CXXFLAGS += -DPAPI_MEM
75+
testpackage: CXXFLAGS += -DPAPI_MEM
76+
77+
#======== Allocator =========
78+
#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
79+
#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
80+
CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
81+
testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
82+
83+
84+
# BOOST_VERSION = current trilinos version
85+
# ZOLTAN_VERSION = current trilinos verson
86+
#
87+
#======== Libraries ===========
88+
89+
MPT_BRAND = OpenMPI
90+
MPT_VERSION = 4.1.1
91+
92+
JEMALLOC_VERSION = 5.2.1
93+
LIBRARY_PREFIX = /proj/group/spacephysics/libraries
94+
95+
#compiled libraries mostly in modules
96+
LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/phiprof/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/phiprof/lib
97+
INC_PROFILE = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/phiprof/include
98+
99+
LIB_VLSV = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/vlsv
100+
INC_VLSV = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/vlsv
101+
102+
LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/jemalloc/lib -ljemalloc -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/jemalloc/lib
103+
INC_JEMALLOC = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/jemalloc/include
104+
105+
LIB_BOOST = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/boost/lib -lboost_program_options -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/boost/lib
106+
INC_BOOST = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/boost/include
107+
108+
LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/zoltan/lib -lzoltan -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/zoltan/lib
109+
INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/zoltan/include
110+
111+
LIB_PAPI = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/papi/lib
112+
INC_PAPI = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/papi/include
113+
114+
115+
#header libraries
116+
INC_EIGEN = -I$(LIBRARY_PREFIX)/ -I$(LIBRARY_PREFIX)/Eigen/
117+
INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/
118+
INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/
119+
INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass/
120+
121+
122+
123+

0 commit comments

Comments
 (0)