|
1 | | -using Conda |
| 1 | +using PyCall |
2 | 2 |
|
3 | 3 | dpro_repo = get(ENV, "DEVITO_PRO", "") |
4 | 4 | which_devito = get(ENV,"DEVITO_BRANCH", "") |
5 | | -try |
6 | 5 |
|
7 | | - Conda.pip_interop(true) |
8 | | - # optional devito pro installation |
9 | | - # note that the submodules do not install correctly from a URL, so we need install from local cloned repo |
10 | | - # 2024-07-29 this is totally hacked for nvidia HPCX and Open MPI 4.1.7a1 |
11 | | - # using nvidia HPC SDK 24.7 and cuda 12.5 |
12 | | - if dpro_repo != "" |
13 | | - @info "Building devitopro from latest release" |
14 | | - Conda.pip("uninstall -y", "devitopro") |
15 | | - Conda.pip("uninstall -y", "devito") |
16 | | - Conda.pip("uninstall -y", "numpy") |
17 | | - Conda.pip("uninstall -y", "mpi4py") |
18 | | - |
19 | | - # clone the devitopro repository and init submodules |
20 | | - dir = "$(tempname())-devitopro" |
21 | | - _pwd = pwd() |
22 | | - Sys.which("git") === nothing && error("git is not installed") |
23 | | - run(`git clone $(dpro_repo) $(dir)`) |
24 | | - cd(dir) |
25 | | - run(`git submodule update --init`) |
26 | | - cd(_pwd) |
| 6 | +# Check if packages altready installed |
| 7 | +# The assumption is that if the packages are already installed, the user |
| 8 | +# has already set up the environment and we don't need to do anything |
27 | 9 |
|
28 | | - # get DEVITO_ARCH if it exists, default to gcc |
29 | | - devito_arch = get(ENV, "DEVITO_ARCH", "gcc") |
30 | | - |
31 | | - # devito requirements |
32 | | - Conda.pip("install --no-cache-dir -r", "https://raw.githubusercontent.com/devitocodes/devito/master/requirements.txt") |
| 10 | +# First thing first, is devito already installed |
| 11 | +devito = try |
| 12 | + pyimport("devito") |
| 13 | + which_devito == "" |
| 14 | +catch e |
| 15 | + @info "Devito not installed or broken" |
| 16 | + false |
| 17 | +end |
33 | 18 |
|
34 | | - if lowercase(devito_arch) == "nvc" |
35 | | - ENV["CC"] = "nvc" |
36 | | - ENV["CFLAGS"] = "-noswitcherror -tp=px" |
37 | | - elseif lowercase(devito_arch) == "gcc" |
38 | | - ENV["CC"] = "gcc" |
39 | | - ENV["CFLAGS"] = "" |
40 | | - end |
41 | | - @info "DEVITO_ARCH=$(devito_arch)" |
42 | | - @info "CC=$(ENV["CC"])" |
43 | | - @info "CFLAGS=$(ENV["CFLAGS"])" |
| 19 | +# Second, is devitopro installed |
| 20 | +devitopro = try |
| 21 | + pyimport("devitopro") |
| 22 | + true |
| 23 | +catch e |
| 24 | + @info "DevitoPRO not installed or broken" |
| 25 | + dpro_repo == "" |
| 26 | +end |
44 | 27 |
|
45 | | - # devitopro |
46 | | - Conda.pip("install", "$(dir)") |
47 | | - rm(dir, recursive=true, force=true) |
| 28 | +if (devito && devitopro) |
| 29 | + @info "Devito and DevitoPRO are already installed, no need to build" |
| 30 | + return |
| 31 | +end |
48 | 32 |
|
49 | | - # nvida requirements |
50 | | - if lowercase(devito_arch) == "nvc" |
51 | | - Conda.pip("install --no-cache-dir -r", "https://raw.githubusercontent.com/devitocodes/devito/master/requirements-nvidia.txt") |
52 | | - end |
| 33 | +# Setup pip command. This will automatically pickup whichever pip PyCall is setup with. |
| 34 | +function pip(pkg::String) |
| 35 | + cmd_args = Vector{String}([PyCall.python, "-m", "pip", "install", "--no-cache-dir", split(pkg, " ")...]) |
| 36 | + run(Cmd(cmd_args)) |
| 37 | +end |
53 | 38 |
|
54 | | - # mpi requirements |
55 | | - Conda.pip("install --no-cache-dir -r", "https://raw.githubusercontent.com/devitocodes/devito/master/requirements-mpi.txt") |
| 39 | +# MPI4PY and optional nvidia requirements |
| 40 | +function mpi4py(mpireqs) |
| 41 | + try |
| 42 | + ENV["CC"] = "nvc" |
| 43 | + ENV["CFLAGS"] = "-noswitcherror -tp=px" |
| 44 | + pip("-r $(mpireqs)requirements-mpi.txt") |
| 45 | + # If this succeeded, the we might need the extra nvidia python requirements |
| 46 | + pip("-r $(mpireqs)requirements-nvidia.txt") |
| 47 | + catch e |
| 48 | + # Default. Don't set any flag an use the default compiler |
56 | 49 | delete!(ENV,"CFLAGS") |
| 50 | + ENV["CC"] = "gcc" |
| 51 | + pip("-r $(mpireqs)requirements-mpi.txt") |
| 52 | + end |
| 53 | + delete!(ENV,"CFLAGS") |
| 54 | + delete!(ENV,"CC") |
| 55 | +end |
57 | 56 |
|
58 | | - elseif which_devito != "" |
59 | | - @info "Building devito from branch $(which_devito)" |
60 | | - Conda.pip("install", "devito[tests,extras,mpi]@git+https://github.com/devitocodes/devito@$(which_devito)") |
61 | | - |
62 | | - else |
63 | | - @info "Building devito from latest release" |
64 | | - Conda.pip("uninstall -y", "devitopro") |
65 | | - Conda.pip("uninstall -y", "devito") |
66 | | - |
67 | | - dir = "$(tempname())-devito" |
| 57 | +# Install devito and devitopro |
| 58 | +try |
| 59 | + # Some python version don't like without --user so bypass it |
| 60 | + ENV["PIP_BREAK_SYSTEM_PACKAGES"] = "1" |
| 61 | + if dpro_repo != "" |
| 62 | + # Devitopro is available (Licensed). Install devitopro that comes with devito |
| 63 | + # as a submodule. THis way we install the devito version that is compatible with devitopro |
| 64 | + # and we don't need to install devito separately |
| 65 | + # Because devito is a submodule, pip fails to install it properly (pip does not clone with --recursive) |
| 66 | + # So we need to clone then install it. And since julia somehow doesn't think submodules exists LibGit2 cannot clone |
| 67 | + # the submodules. So we need to clone it with git by hand |
| 68 | + dir = "$(tempname())-devitopro" |
68 | 69 | Sys.which("git") === nothing && error("git is not installed") |
69 | | - run(`git clone https://github.com/devitocodes/devito $(dir)`) |
70 | | - |
71 | | - Conda.pip("install", "$(dir)[tests,extras]") |
| 70 | + run(`git clone --recurse-submodules --depth 1 $(dpro_repo) $(dir)`) |
| 71 | + |
| 72 | + # Install devitopro |
| 73 | + pip(dir) |
| 74 | + |
| 75 | + # Now all we need is mpi4py. It is straightforward to install except with the nvidia compiler that requires |
| 76 | + # extra flags to ignore some flags set by mpi4py |
| 77 | + mpi4py("$(dir)/submodules/devito/") |
72 | 78 | rm(dir, recursive=true, force=true) |
73 | | - |
74 | | - # Conda.pip("install", "$(dir)[tests,extras,mpi]") |
75 | | - # ENV["CC"] = "gcc" |
76 | | - # ENV["CFLAGS"] = "" |
77 | | - # ENV["MPICC"] = "mpicc" |
78 | | - # Conda.pip("uninstall -y", "mpi4py") |
79 | | - # Conda.pip("install", "mpi4py") |
| 79 | + |
| 80 | + # Make sure it imports |
| 81 | + pyimport("devitopro") |
| 82 | + pyimport("devito") |
| 83 | + else |
| 84 | + if which_devito != "" |
| 85 | + @info "Building devito from branch $(which_devito)" |
| 86 | + pip("devito[extras,tests]@git+https://github.com/devitocodes/devito@$(which_devito)") |
| 87 | + mpi4py("https://raw.githubusercontent.com/devitocodes/devito/$(which_devito)/") |
| 88 | + else |
| 89 | + @info "Building devito from latest release" |
| 90 | + pip("devito[extras,tests]") |
| 91 | + mpi4py("https://raw.githubusercontent.com/devitocodes/devito/main/") |
| 92 | + end |
| 93 | + # Make sure it imports |
| 94 | + pyimport("devito") |
80 | 95 | end |
| 96 | + delete!(ENV, "PIP_BREAK_SYSTEM_PACKAGES") |
81 | 97 | catch e |
82 | 98 | if get(ENV, "JULIA_REGISTRYCI_AUTOMERGE", "false") == "true" |
83 | 99 | @warn "unable to build" |
|
0 commit comments