Skip to content

Commit 9e8bafa

Browse files
authored
Verify environment variables (#94)
1 parent 0518487 commit 9e8bafa

File tree

22 files changed

+390
-138
lines changed

22 files changed

+390
-138
lines changed

.bandit.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,12 +83,12 @@
8383
# IPAS Required Checkers. Do not disable these
8484
# Additional checkers may be added if desired
8585
tests:
86-
[ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B320', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B404', 'B405', 'B406', 'B407', 'B408', 'B409', 'B410', 'B411', 'B412', 'B413']
86+
[ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B320', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B405', 'B406', 'B407', 'B408', 'B409', 'B410', 'B411', 'B412', 'B413']
8787

8888
# (optional) list skipped test IDs here, eg '[B101, B406]':
8989
# The following checkers are not required but be added to tests list if desired
9090
skips:
91-
[ 'B101', 'B102', 'B103', 'B104', 'B105', 'B106', 'B107', 'B108', 'B110', 'B112', 'B201', 'B501', 'B502', 'B503', 'B504', 'B505', 'B506', 'B507', 'B601', 'B602', 'B603', 'B604', 'B605', 'B606', 'B607', 'B608', 'B609', 'B610', 'B611', 'B701', 'B702', 'B703']
91+
[ 'B101', 'B102', 'B103', 'B104', 'B105', 'B106', 'B107', 'B108', 'B110', 'B112', 'B201', 'B404', 'B501', 'B502', 'B503', 'B504', 'B505', 'B506', 'B507', 'B601', 'B602', 'B603', 'B604', 'B605', 'B606', 'B607', 'B608', 'B609', 'B610', 'B611', 'B701', 'B702', 'B703']
9292

9393
### (optional) plugin settings - some test plugins require configuration data
9494
### that may be given here, per-plugin. All bandit test plugins have a built in

.github/workflows/ci.yml

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,16 @@ jobs:
2626
token: ${{ secrets.PATIMEX }}
2727
path: third_party/imex
2828
ref: ${{ env.IMEX_SHA }}
29-
- name: Cache miniconda
30-
id: cache-miniconda
31-
uses: actions/cache@v3
32-
env:
33-
MINICONDA_CACHE_NUMBER: 2 # Increase to reset cache
34-
with:
35-
path: third_party/install/miniconda/**
36-
key: ${{ runner.os }}-miniconda-${{ env.MINICONDA_CACHE_NUMBER }}-${{ hashFiles('conda-env.txt') }}
29+
# - name: Cache miniconda
30+
# id: cache-miniconda
31+
# uses: actions/cache@v3
32+
# env:
33+
# MINICONDA_CACHE_NUMBER: 2 # Increase to reset cache
34+
# with:
35+
# path: third_party/install/miniconda/**
36+
# key: ${{ runner.os }}-miniconda-${{ env.MINICONDA_CACHE_NUMBER }}-${{ hashFiles('conda-env.txt') }}
3737
- name: Miniconda
38-
if: steps.cache-miniconda.outputs.cache-hit != 'true'
38+
# if: steps.cache-miniconda.outputs.cache-hit != 'true'
3939
run: |
4040
rm -rf "$GITHUB_WORKSPACE"/third_party/install/miniconda
4141
mkdir -p "$GITHUB_WORKSPACE"/third_party/install
@@ -50,12 +50,12 @@ jobs:
5050
cd -
5151
conda create --file conda-env.txt --name sharpy
5252
conda clean -a -y
53-
- name: Setup miniconda
54-
if: steps.cache-miniconda.outputs.cache-hit == 'true'
55-
run: |
56-
echo "$GITHUB_WORKSPACE/third_party/install/miniconda/bin" >> $GITHUB_PATH
57-
echo "$GITHUB_WORKSPACE/third_party/install/miniconda/condabin" >> $GITHUB_PATH
58-
export PATH=$GITHUB_WORKSPACE/third_party/install/miniconda/bin:${PATH}
53+
# - name: Setup miniconda
54+
# if: steps.cache-miniconda.outputs.cache-hit == 'true'
55+
# run: |
56+
# echo "$GITHUB_WORKSPACE/third_party/install/miniconda/bin" >> $GITHUB_PATH
57+
# echo "$GITHUB_WORKSPACE/third_party/install/miniconda/condabin" >> $GITHUB_PATH
58+
# export PATH=$GITHUB_WORKSPACE/third_party/install/miniconda/bin:${PATH}
5959
- name: Setup LLVM Cache Var
6060
run: |
6161
echo 'LLVM_SHA<<EOF' >> $GITHUB_ENV

README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ If your compiler does not default to a recent (e.g. g++ >= 9) version, try somet
3434
# single rank
3535
pytest test
3636
# distributed on multiple ($N) ranks/processes
37-
SHARPY_IDTR_SO=`pwd`/sharpy/libidtr.so mpirun -n $N python -m pytest test
37+
mpirun -n $N python -m pytest test
3838
```
3939

4040
## Running
@@ -57,13 +57,13 @@ sp.fini()
5757
Assuming the above is in file `simple.py` a single-process run is executed like
5858

5959
```bash
60-
SHARPY_IDTR_SO=`pwd`/sharpy/libidtr.so python simple.py
60+
python simple.py
6161
```
6262

6363
and multi-process run is executed like
6464

6565
```bash
66-
SHARPY_IDTR_SO=`pwd`/sharpy/libidtr.so mpirun -n 5 python simple.py
66+
mpirun -n 5 python simple.py
6767
```
6868

6969
### Distributed Execution without mpirun
@@ -76,7 +76,7 @@ Additionally SHARPY_MPI_HOSTS can be used to control the host to use for spawnin
7676
The following command will run the stencil example on 3 MPI ranks:
7777

7878
```bash
79-
SHARPY_IDTR_SO=`pwd`/sharpy/libidtr.so \
79+
SHARPY_FALLBACK=numpy \
8080
SHARPY_MPI_SPAWN=2 \
8181
SHARPY_MPI_EXECUTABLE=`which python` \
8282
SHARPY_MPI_EXE_ARGS="examples/stencil-2d.py 10 2000 star 2" \

examples/black_scholes.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ def run(nopt, backend, iterations, datatype):
176176
import sharpy as np
177177
from sharpy import fini, init, sync
178178

179-
device = os.getenv("SHARPY_USE_GPU", "")
179+
device = os.getenv("SHARPY_DEVICE", "")
180180
create_full = partial(np.full, device=device)
181181
erf = np.erf
182182

examples/shallow_water.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def run(n, backend, datatype, benchmark_mode):
5555
from sharpy import fini, init, sync
5656
from sharpy.numpy import fromfunction as _fromfunction
5757

58-
device = os.getenv("SHARPY_USE_GPU", "")
58+
device = os.getenv("SHARPY_DEVICE", "")
5959
create_full = partial(np.full, device=device)
6060
fromfunction = partial(_fromfunction, device=device)
6161

examples/stencil-2d.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ def main():
118118
print("Data type = double precision")
119119
print("Compact representation of stencil loop body")
120120

121-
device = os.getenv("SHARPY_USE_GPU", "")
121+
device = os.getenv("SHARPY_DEVICE", "")
122122

123123
# there is certainly a more Pythonic way to initialize W,
124124
# but it will have no impact on performance.

examples/wave_equation.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def run(n, backend, datatype, benchmark_mode):
5555
from sharpy import fini, init, sync
5656
from sharpy.numpy import fromfunction as _fromfunction
5757

58-
device = os.getenv("SHARPY_USE_GPU", "")
58+
device = os.getenv("SHARPY_DEVICE", "")
5959
create_full = partial(np.full, device=device)
6060
fromfunction = partial(_fromfunction, device=device)
6161

sharpy/__init__.py

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
# At this point there are no checks of input arguments whatsoever, arguments
1414
# are simply forwarded as-is.
1515

16+
import os
17+
import re
1618
from importlib import import_module
1719
from os import getenv
1820
from typing import Any
@@ -42,8 +44,11 @@
4244

4345

4446
def init(cw=None):
47+
libidtr = os.path.join(os.path.dirname(__file__), "libidtr.so")
48+
assert os.path.isfile(libidtr), "libidtr.so not found"
49+
4550
cw = _sharpy_cw if cw is None else cw
46-
_init(cw)
51+
_init(cw, libidtr)
4752

4853

4954
def to_numpy(a):
@@ -64,31 +69,42 @@ def to_numpy(a):
6469
f"{op} = lambda this: ndarray(_csp.EWUnyOp.op(_csp.{OP}, this._t))"
6570
)
6671

72+
73+
def _validate_device(device):
74+
if len(device) == 0 or re.search(
75+
r"^((opencl|level-zero|cuda):)?(host|gpu|cpu|accelerator)(:\d+)?$",
76+
device,
77+
):
78+
return device
79+
else:
80+
raise ValueError(f"Invalid device string: {device}")
81+
82+
6783
for func in api.api_categories["Creator"]:
6884
FUNC = func.upper()
6985
if func == "full":
7086
exec(
71-
f"{func} = lambda shape, val, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, val, dtype, device, team))"
87+
f"{func} = lambda shape, val, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, val, dtype, _validate_device(device), team))"
7288
)
7389
elif func == "empty":
7490
exec(
75-
f"{func} = lambda shape, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, None, dtype, device, team))"
91+
f"{func} = lambda shape, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, None, dtype, _validate_device(device), team))"
7692
)
7793
elif func == "ones":
7894
exec(
79-
f"{func} = lambda shape, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, 1, dtype, device, team))"
95+
f"{func} = lambda shape, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, 1, dtype, _validate_device(device), team))"
8096
)
8197
elif func == "zeros":
8298
exec(
83-
f"{func} = lambda shape, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, 0, dtype, device, team))"
99+
f"{func} = lambda shape, dtype=float64, device='', team=1: ndarray(_csp.Creator.full(shape, 0, dtype, _validate_device(device), team))"
84100
)
85101
elif func == "arange":
86102
exec(
87-
f"{func} = lambda start, end, step, dtype=int64, device='', team=1: ndarray(_csp.Creator.arange(start, end, step, dtype, device, team))"
103+
f"{func} = lambda start, end, step, dtype=int64, device='', team=1: ndarray(_csp.Creator.arange(start, end, step, dtype, _validate_device(device), team))"
88104
)
89105
elif func == "linspace":
90106
exec(
91-
f"{func} = lambda start, end, step, endpoint, dtype=float64, device='', team=1: ndarray(_csp.Creator.linspace(start, end, step, endpoint, dtype, device, team))"
107+
f"{func} = lambda start, end, step, endpoint, dtype=float64, device='', team=1: ndarray(_csp.Creator.linspace(start, end, step, endpoint, dtype, _validate_device(device), team))"
92108
)
93109

94110
for func in api.api_categories["ReduceOp"]:
@@ -116,10 +132,17 @@ def to_numpy(a):
116132

117133
_fb_env = getenv("SHARPY_FALLBACK")
118134
if _fb_env is not None:
135+
if not _fb_env.isalnum():
136+
raise ValueError(f"Invalid SHARPY_FALLBACK value '{_fb_env}'")
119137

120138
class _fallback:
121139
"Fallback to whatever is provided in SHARPY_FALLBACK"
122-
_fb_lib = import_module(_fb_env)
140+
try:
141+
_fb_lib = import_module(_fb_env)
142+
except ModuleNotFoundError:
143+
raise ValueError(
144+
f"Invalid SHARPY_FALLBACK value '{_fb_env}': module not found"
145+
)
123146

124147
def __init__(self, fname: str, mod=None) -> None:
125148
"""get callable with name 'fname' from fallback-lib

src/Creator.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
namespace SHARPY {
2424

25-
static const char *FORCE_DIST = getenv("SHARPY_FORCE_DIST");
25+
static bool FORCE_DIST = get_bool_env("SHARPY_FORCE_DIST");
2626

2727
inline uint64_t mkTeam(uint64_t team) {
2828
if (team && (FORCE_DIST || getTransceiver()->nranks() > 1)) {

src/Deferred.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,15 +81,15 @@ void Runable::fini() { _deferred.clear(); }
8181
// When execution is needed, the function signature (input args, return
8282
// statement) is finalized, the function gets compiled and executed. The loop
8383
// completes by calling run() on the requesting object.
84-
void process_promises() {
84+
void process_promises(const std::string &libidtr) {
8585
int vtProcessSym, vtSHARPYClass, vtPopSym;
8686
VT(VT_classdef, "sharpy", &vtSHARPYClass);
8787
VT(VT_funcdef, "process", vtSHARPYClass, &vtProcessSym);
8888
VT(VT_funcdef, "pop", vtSHARPYClass, &vtPopSym);
8989
VT(VT_begin, vtProcessSym);
9090

9191
bool done = false;
92-
jit::JIT jit;
92+
jit::JIT jit(libidtr);
9393
std::vector<Runable::ptr_type> deleters;
9494

9595
do {

0 commit comments

Comments
 (0)