diff --git a/.gitignore b/.gitignore index 2f424f5..d5d35bf 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,8 @@ libltdl/ /config/lt~obsolete.m4 /config/ar-lib /config/tap-driver.sh - +/config/tap-driver.py +/config/py-compile # docs intermediate files /doc/man*/*.xml /doc/_build diff --git a/Makefile.am b/Makefile.am index 1f8eb14..89b841e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,5 @@ +.NOTPARALLEL: -SUBDIRS = src test-suite etc +SUBDIRS = src test-suite etc bindings -ACLOCAL_AMFLAGS = -I config +ACLOCAL_AMFLAGS = -I config \ No newline at end of file diff --git a/bindings/Makefile.am b/bindings/Makefile.am new file mode 100644 index 0000000..e6e35a0 --- /dev/null +++ b/bindings/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = python \ No newline at end of file diff --git a/bindings/README.md b/bindings/README.md new file mode 100644 index 0000000..af0ac19 --- /dev/null +++ b/bindings/README.md @@ -0,0 +1,17 @@ +## Bindings for mpibind + +### Overview + +In order to improve its usability, we offer bindings for the mpibind library in +languagues beyond C. Currently the only other language supported is Python. + +### License + +*mpibind* is distributed under the terms of the MIT license. All new +contributions must be made under this license. + +See [LICENSE](LICENSE) and [NOTICE](NOTICE) for details. + +SPDX-License-Identifier: MIT. + +LLNL-CODE-812647. diff --git a/bindings/python/Makefile.am b/bindings/python/Makefile.am new file mode 100644 index 0000000..0214539 --- /dev/null +++ b/bindings/python/Makefile.am @@ -0,0 +1,19 @@ +.NOTPARALLEL: + +SUBDIRS= _mpibind mpibind + +AM_CPPFLAGS = \ + -Wall -Werror -Wno-missing-field-initializers \ + -I$(top_srcdir) -I$(top_srcdir)/src/ -L$(top_srcdir)/src/.libs \ + $(PYTHON_CPPFLAGS) + +all-local: + (cd $(srcdir); CFLAGS="$(AM_CPPFLAGS)" \ + $(PYTHON) setup.py bdist_wheel) + +clean-local: + -rm -f *.pyc *.pyo + -rm -rf __pycache__ + -rm -rf mpibind.egg-info + -rm -rf build + -rm -rf dist \ No newline at end of file diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 0000000..afd9b45 --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,25 @@ +## Python Bindings for mpibind + +### Overview + +You can interact with mpibind through your python scripts: + +```python +# running on syrah login node +from mpibind import MpibindWrapper +wrapper = MpibindWrapper() +mapping = wrapper.get_mapping(ntasks=4) +``` + +### Installing + +### License + +*mpibind* is distributed under the terms of the MIT license. All new +contributions must be made under this license. + +See [LICENSE](LICENSE) and [NOTICE](NOTICE) for details. + +SPDX-License-Identifier: MIT. + +LLNL-CODE-812647. diff --git a/bindings/python/_mpibind/Makefile.am b/bindings/python/_mpibind/Makefile.am new file mode 100644 index 0000000..20a3a55 --- /dev/null +++ b/bindings/python/_mpibind/Makefile.am @@ -0,0 +1,54 @@ +AM_CPPFLAGS = \ + -Wall -Werror -Wno-missing-field-initializers \ + -I$(top_srcdir) -I$(top_srcdir)/src/ \ + $(PYTHON_CPPFLAGS) + +AM_LDFLAGS = \ + -avoid-version -module -Wl,-rpath,$(PYTHON_PREFIX)/lib + +_pympibind.c: $(srcdir)/build.py _mpibind_preproc.h + $(PYTHON) $< + +# Flux LICENSE +_mpibind_preproc.h: _mpibind_clean.h + $(CC) -E '-D__attribute__(...)=' _mpibind_clean.h\ + | sed -e '/^# [0-9]*.*/d' > $@ + +_mpibind_clean.h: Makefile + $(PYTHON) $(srcdir)/clean_header.py \ + $(top_builddir)/src/mpibind.h \ + _mpibind_clean.h + +BUILT_SOURCES= _pympibind.c _mpibind_preproc.h +mpibindpyso_LTLIBRARIES = _pympibind.la +mpibindpyso_PYTHON = __init__.py + +nodist_mpibindbindinginclude_HEADERS = _mpibind_preproc.h +nodist__pympibind_la_SOURCES = _pympibind.c + +common_libs = $(top_builddir)/src/libmpibind.la \ + $(PYTHON_LDFLAGS) + +_pympibind_la_LIBADD = $(common_libs) + +EXTRA_DIST=build.py make_clean_header.py + +clean-local: + -rm -f *.c *.so *.pyc *.pyo *_clean.h *_preproc.h + -rm -rf __pycache__ + -rm -rf .libs + +# creates a symbolic link _pympibind.so to +# _pympibind.so in _mpibind/.libs so that it can be imported in mpibind/wrapper +# Flux LICENSE + +.PHONY: lib-copy + +lib-copy: ${mpibindpyso_PYTHON} ${mpibindpyso_LTLIBRARIES} + -echo Copying libraries to where they can be used by python in-tree + for LIB in ${mpibindpyso_LTLIBRARIES:la=so} ; do \ + test -e .libs/$$LIB && \ + $(LN_S) .libs/$$LIB ./ || true; \ + done + +all-local: lib-copy \ No newline at end of file diff --git a/bindings/python/_mpibind/__init__.py b/bindings/python/_mpibind/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bindings/python/_mpibind/build.py b/bindings/python/_mpibind/build.py new file mode 100644 index 0000000..173d3ac --- /dev/null +++ b/bindings/python/_mpibind/build.py @@ -0,0 +1,36 @@ +from cffi import FFI +from os import path + +ffibuilder = FFI() + +# add some of the opaque hwloc objects +cdefs = """ +typedef struct { ...; } hwloc_topology_t; +typedef struct { ...; } hwloc_bitmap_t; +""" + +# this file is used from two different places during build +# and then during distribution +prepared_headers = "_mpibind_preproc.h" +if not path.exists(prepared_headers): + prepared_headers = "_mpibind/_mpibind_preproc.h" + +# add cleaned header file to the definitions we expose to python +with open(prepared_headers) as h: + cdefs = cdefs + h.read() + +# set the defintions we expose to python +ffibuilder.cdef(cdefs) + +# indicate that the library makes the definitions we set +ffibuilder.set_source( + "_mpibind._pympibind", + """ + #include + """, + libraries=["mpibind"], +) + +#emit c code and let autotools build _pympibind.so extension module +if __name__ == "__main__": + ffibuilder.emit_c_code("_pympibind.c") \ No newline at end of file diff --git a/bindings/python/_mpibind/clean_header.py b/bindings/python/_mpibind/clean_header.py new file mode 100644 index 0000000..7194b76 --- /dev/null +++ b/bindings/python/_mpibind/clean_header.py @@ -0,0 +1,56 @@ +import os +import re +import struct + +import argparse + +platform_c_maxint = 2 ** (struct.Struct('i').size * 8 - 1) - 1 + +def parse_arguments(): + parser = argparse.ArgumentParser() + + parser.add_argument("header", help="header file to parse", type=str) + parser.add_argument("output", help="destination of clean header", type=str) + + return parser.parse_args() + +def apply_replacement_rules(line): + rules = { + "__restrict": "restrict", + "__inline__": "inline", + "INT_MAX": str(platform_c_maxint), + } + + for k,v in rules.items(): + line = line.replace(k, v) + + if ("inline" in line): + print(line) + return line + +def make_enum_literal(line): + return re.sub(r'(\d)(UL)?<<(\d)',lambda x: str(int(x.group(1))<', '', line) + +def clean_header(header): + cleaned = "" + with open(header, "r") as f: + for line in f.readlines(): + line = apply_replacement_rules(line) + line = make_enum_literal(line) + line = remove_external_headers(line) + cleaned += line + + return cleaned + +def output_header(output_file, cleaned): + with open(output_file, "w") as f: + f.write(cleaned) + +if __name__ == "__main__": + args = parse_arguments() + abs_header = os.path.abspath(args.header) + abs_output = os.path.abspath(args.output) + output_header(abs_output, clean_header(abs_header)) \ No newline at end of file diff --git a/bindings/python/mpibind/Makefile.am b/bindings/python/mpibind/Makefile.am new file mode 100644 index 0000000..68081b4 --- /dev/null +++ b/bindings/python/mpibind/Makefile.am @@ -0,0 +1,7 @@ +mpibindpy_PYTHON=\ + __init__.py\ + wrapper.py + +clean-local: + -rm -f *.pyc *.pyo + -rm -rf __pycache__ \ No newline at end of file diff --git a/bindings/python/mpibind/__init__.py b/bindings/python/mpibind/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bindings/python/mpibind/wrapper.py b/bindings/python/mpibind/wrapper.py new file mode 100644 index 0000000..226471c --- /dev/null +++ b/bindings/python/mpibind/wrapper.py @@ -0,0 +1,161 @@ +import os +import re + +from _mpibind._pympibind import ffi, lib + +class MpibindWrapper(): + def __init__(self): + self.ffi = ffi + self.lib = lib + + def get_mapping( + self, + handle=None, + topology=None, + **kwargs + ): + if topology is not None: + self.use_topology(topology) + + if handle is None: + p_handle = self.ffi.new("struct mpibind_t **") + self.lib.mpibind_init(p_handle) + handle = p_handle[0] + + self._configure_handle(handle, **kwargs) + self.mpibind(handle) + return MpibindMapping(self._get_mapping_string(handle)) + + def use_topology(self, topology_file_path): + os.environ['HWLOC_XMLFILE'] = topology_file_path + + def set_ntasks(self, handle, ntasks): + self.lib.mpibind_set_ntasks(handle, ntasks) + + def set_nthreads(self, handle, nthreads): + self.lib.mpibind_set_nthreads(handle, nthreads) + + def set_smt(self, handle, smt): + self.lib.mpibind_set_smt(handle, smt) + + def set_gpu_optim(self, handle, gpu_optim): + self.lib.mpibind_set_gpu_optim(handle, gpu_optim) + + def set_greedy(self, handle, greedy): + self.lib.mpibind_set_greedy(handle, greedy) + + def mpibind(self, handle): + self.lib.mpibind(handle) + + def _get_mapping_string(self, handle, buffer_size = 2048, encoding = "utf-8"): + mapping_buffer = self.ffi.new("char[]", buffer_size) + self.lib.mpibind_get_mapping_string(handle, mapping_buffer, buffer_size) + return self.ffi.string(mapping_buffer).decode(encoding) + + def _configure_handle(self, handle, **kwargs): + if 'ntasks' in kwargs: + self.set_ntasks(handle, kwargs['ntasks']) + if 'nthreads' in kwargs: + self.set_nthreads(handle, kwargs['nthreads']) + if 'smt' in kwargs: + self.set_smt(handle, kwargs['smt']) + if 'gpu_optim' in kwargs: + self.set_gpu_optim(handle, kwargs['gpu_optim']) + if 'greedy' in kwargs: + self.set_greedy(handle, kwargs['greedy']) + +class MpibindMapping(): + def __init__(self, mapping_string): + self.assignments = [] + self._parse_mapping(mapping_string) + + def _parse_mapping(self, mapping_string): + mapping_pattern = r'task([\d\s]+)thds([\d\s]+)gpus([\d\s]+)cpus([\d\s,-]+)' + mapping_string = mapping_string.replace('\n', '').split('mpibind:')[1:] + for line in mapping_string: + result = re.search(mapping_pattern, line) + thds = result.group(2).strip() + gpus = result.group(3).strip() + cpus = result.group(4).strip() + + self.assignments.append(MpibindTaskAssignment(cpus, gpus, thds)) + + def __repr__(self): + rep = "" + for idx, member in enumerate(self.assignments): + rep += f"task {idx}: {repr(member)}\n" + return rep + + def __getitem__(self, idx): + return self.assignments[idx] + +class MpibindTaskAssignment(): + def __init__(self, cpus, gpus, thread_count): + self.cpus = MpibindResourceAssignment("cpus", cpus) + self.gpus = MpibindResourceAssignment("gpus", gpus) + self.smt = thread_count + + def __repr__(self): + return f"smt: {self.smt} {repr(self.gpus)} {repr(self.cpus)}" + + +class MpibindResourceAssignment(): + def __init__(self, resource_type, resource_string): + self.type = resource_type + self.assignment = resource_string + + @property + def assignment(self): + return self.__assignment + + @assignment.setter + def assignment(self, assignment): + self.__assignment = assignment + self.count = self._parse_count(assignment) + + def _parse_count(self, assignment): + if not assignment: + return 0 + + count = 0 + for item in assignment.split(","): + count += 1 + if "-" in item: + start, end = item.split("-") + count += (int(end) - int(start)) + + return count + + def __repr__(self): + return f"{self.type}: {self.assignment}" + + +if __name__ == "__main__": + """Demontration of working with mpibind python wrapper""" + wrapper = MpibindWrapper() + + #Work directly with ffi through wrapper + p_handle = wrapper.ffi.new("struct mpibind_t **") + wrapper.lib.mpibind_init(p_handle) + handle = p_handle[0] + wrapper.lib.mpibind_set_ntasks(handle, 4) + wrapper.lib.mpibind(handle) + buffer = wrapper.ffi.new("char[]", 2048) + wrapper.lib.mpibind_get_mapping_string(handle, buffer, 2048) + low_mapping = MpibindMapping(wrapper.ffi.string(buffer).decode('utf-8')) + + print("Working with cffi:") + print(low_mapping) + print() + + #Let wrapper parse mapping with handle configured using direct ffi + mid_mapping = wrapper.get_mapping(handle=handle) + print("Passing pre-configured handle:") + print(low_mapping) + print() + + #Letting the wrapper use a one-time handle + high_mapping = wrapper.get_mapping(ntasks=4) + print("Letting wrapper use one-time handle:") + print(low_mapping) + print() \ No newline at end of file diff --git a/bindings/python/setup.py b/bindings/python/setup.py new file mode 100644 index 0000000..5410db1 --- /dev/null +++ b/bindings/python/setup.py @@ -0,0 +1,21 @@ +from setuptools import setup +import os + +here = os.path.abspath(os.path.dirname(__file__)) +cffi_dep = "cffi>=1.0.0" +setup( + name="mpibind", + version="0.0.1", + description="Python bindings for mpibind", + setup_requires=[cffi_dep], + cffi_modules=["_mpibind/build.py:ffibuilder"], + install_requires=[cffi_dep], + url="https://github.com/LLNL/mpibind", + author="", + author_email="", + license="", + package_dir={ + "": here, + }, + packages=["_mpibind", "mpibind"] +) diff --git a/cmd/example.py b/cmd/example.py new file mode 100644 index 0000000..d203d01 --- /dev/null +++ b/cmd/example.py @@ -0,0 +1,55 @@ +from _mpibind._mpibind import ffi, lib +from .wrapper import MissingFunctionError +from .mpibind_wrapper import MpibindWrapper + +ntasks = 4 +nthreads = 3 +greedy = 0 +gpu_optim = 0 +smt_level = 1 + + +def set_handle(handle, ntasks): + lib.mpibind_set_ntasks(handle, ntasks) + +def quick_example_direct(): + handle = ffi.new("struct mpibind_t *") + + set_handle(handle, ntasks) + lib.mpibind_set_nthreads(handle, nthreads) + lib.mpibind_set_greedy(handle, greedy) + lib.mpibind_set_gpu_optim(handle, gpu_optim) + lib.mpibind_set_smt(handle, smt_level) + + if lib.mpibind(handle) != 0: + print("oh no") + + mapping_buffer = ffi.new("char[]", 2048) + lib.mpibind_get_mapping_string(handle, mapping_buffer, 2048) + mapping = ffi.string(mapping_buffer).decode("utf-8") + print(mapping) + + + +def quick_example_wrapper(): + mpibind_py = MpibindWrapper() + handle = ffi.new("struct mpibind_t *") + mpibind_py.set_ntasks(handle, ntasks) + mpibind_py.set_greedy(handle, greedy) + mpibind_py.set_gpu_optim(handle, gpu_optim) + mpibind_py.set_smt(handle, smt_level) + + mpibind_py.use_topology('../../topo-xml/coral-lassen.xml') + + if mpibind_py.mpibind(handle) != 0: + print("failure to create mpibind mapping") + + + print(mapping) + print(mapping.counts) + +def trying_hwloc(): + topo = ffi.new("struct hwloc_topology_t *") + +if __name__ == "__main__": + quick_example_wrapper() \ No newline at end of file diff --git a/cmd/flux-mpibind.py b/cmd/flux-mpibind.py new file mode 100644 index 0000000..40a1fc1 --- /dev/null +++ b/cmd/flux-mpibind.py @@ -0,0 +1,324 @@ +import os +import sys +import logging +import argparse +import json +from itertools import chain + +from _mpibind._mpibind import ffi, lib +from .cffiwrapper.mpibind_wrapper import MpibindWrapper + +import flux +from flux import job +from flux.job import JobspecV1, JobspecV2 +from flux import util +from flux import debugged + + +class RunCmd: + """ + Run multiple jobs on a node using mpibind to divide resources + """ + + def __init__(self, **kwargs): + self.parser = self.run_parser(kwargs) + self.mpibind_py = MpibindWrapper() + + def get_parser(self): + return self.parser + + @staticmethod + def run_parser(exclude_io=False): + """ + Parse Command Line Arguments for jobspec creation + """ + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument( + "--dry-run", + action="store_true", + help="print jobspec instead of submitting jobs", + ) + parser.add_argument( + "-t", + "--time-limit", + type=str, + metavar="FSD", + help="Time limit in Flux standard duration, e.g. 2d, 1.5h", + ) + parser.add_argument( + "--priority", + help="Set job priority (0-31, default=16)", + type=int, + metavar="N", + default=16, + ) + parser.add_argument( + "--setattr", + action="append", + help="Set job attribute ATTR to VAL (multiple use OK)", + metavar="ATTR=VAL", + ) + parser.add_argument( + "--hw-threads-per-core", + "--smt", + type=int, + metavar="N", + help="Number of threads to allocate per core", + default=1 + ) + parser.add_argument( + "-G", + "--greedy", + type=int, + choices={0, 1}, + help="Toggle the greedy option for mpibind", + default=0 + ) + parser.add_argument( + "-g", + "--gpu-optim", + type=int, + choices={0, 1}, + help="Toggle the gpu optimization option for mpibind", + default=0 + ) + parser.add_argument( + "--job-name", + type=str, + help="Set an optional name for job to NAME", + metavar="NAME", + ) + parser.add_argument( + "--command", + help="Job command and arguments", + action="append", + default=[] + ) + parser.add_argument( + "--jobspec-dir", + help="Directory to place jospecs in", + default='.' + ) + parser.add_argument( + "--input", + type=str, + help="Redirect job stdin from FILENAME, bypassing KVS" + if not exclude_io + else argparse.SUPPRESS, + metavar="FILENAME", + ) + parser.add_argument( + "--output", + type=str, + help="Redirect job stdout to FILENAME, bypassing KVS" + if not exclude_io + else argparse.SUPPRESS, + metavar="FILENAME", + ) + parser.add_argument( + "--error", + type=str, + help="Redirect job stderr to FILENAME, bypassing KVS" + if not exclude_io + else argparse.SUPPRESS, + metavar="FILENAME", + ) + parser.add_argument( + "-l", + "--label-io", + action="store_true", + help="Add rank labels to stdout, stderr lines" + if not exclude_io + else argparse.SUPPRESS, + ) + parser.add_argument( + "-o", + "--setopt", + action="append", + help="Set shell option OPT. An optional value is supported with" + + " OPT=VAL (default VAL=1) (multiple use OK)", + metavar="OPT", + ) + parser.add_argument( + "--debug-emulate", action="store_true", help=argparse.SUPPRESS + ) + parser.add_argument( + "--flags", + action="append", + help="Set comma separated list of job submission flags. Possible " + + "flags: debug, waitable", + metavar="FLAGS", + ) + return parser + + def submit(self, args): + """ + Submit job, constructing jobspec from args. + Returns jobid. + """ + flux_handle = flux.Flux() + jids = [] + jobspecs = self.init_jobspec(args) + + for js in jobspecs: + js.cwd = os.getcwd() + js.environment = dict(os.environ) + + if args.time_limit is not None: + js.duration = args.time_limit + + if args.job_name is not None: + js.setattr("system.job.name", args.job_name) + + if args.input is not None: + js.stdin = args.input + + if args.output is not None and args.output not in ["none", "kvs"]: + js.stdout = args.output + + if args.label_io: + js.setattr_shell_option("output.stdout.label", True) + + if args.error is not None: + js.stderr = args.error + if args.label_io: + js.setattr_shell_option("output.stderr.label", True) + + if args.setopt is not None: + for keyval in args.setopt: + # Split into key, val with a default for 1 if no val given: + key, val = (keyval.split("=", 1) + [1])[:2] + try: + val = json.loads(val) + except (json.JSONDecodeError, TypeError): + pass + + js.setattr_shell_option(key, val) + + if args.debug_emulate: + debugged.set_mpir_being_debugged(1) + + if debugged.get_mpir_being_debugged() == 1: + # if stop-tasks-in-exec is present, overwrite + js.setattr_shell_option("stop-tasks-in-exec", json.loads("1")) + + if args.setattr is not None: + for keyval in args.setattr: + tmp = keyval.split("=", 1) + if len(tmp) != 2: + raise ValueError("--setattr: Missing value for attr " + keyval) + key = tmp[0] + try: + val = json.loads(tmp[1]) + except (json.JSONDecodeError, TypeError): + val = tmp[1] + js.setattr(key, val) + + arg_debug = False + arg_waitable = False + if args.flags is not None: + for tmp in args.flags: + for flag in tmp.split(","): + if flag == "debug": + arg_debug = True + elif flag == "waitable": + arg_waitable = True + else: + raise ValueError("--flags: Unknown flag " + flag) + + if args.dry_run: + print(js.dumps(), file=sys.stdout) + else: + jids.append( + job.submit( + flux_handle, + js.dumps(), + priority=args.priority, + waitable=arg_waitable, + debug=arg_debug, + ) + ) + + return jids + + + def init_jobspec(self, args): + """ + generate + """ + num_jobs = len(args.command) + if num_jobs < 1: + print("Please specify at least one job to run") + raise SystemExit + + handle = ffi.new("struct mpibind_t *") + + print( + f""" + You requested: + jobs: {num_jobs} + smt: {args.hw_threads_per_core} + greedy: {args.greedy} + gput_optim: {args.gpu_optim} + """ + + ) + + self.mpibind_py.set_ntasks(handle, num_jobs) + self.mpibind_py.set_greedy(handle, args.greedy) + self.mpibind_py.set_gpu_optim(handle, args.gpu_optim) + self.mpibind_py.set_smt(handle, args.hw_threads_per_core) + self.mpibind_py.mpibind(handle) + + self.mpibind_py.print_mapping(handle) + + mapping = self.mpibind_py.get_mapping(handle) + print(mapping.counts) + jobspecs = [] + for job, counts in zip(args.command, mapping.counts.values()): + jobspecs.append( + JobspecV2.from_command( + job.split(" "), + num_nodes=1, + num_tasks=1, + cores_per_task=counts['cpus'], + hw_threads_per_core= int(counts['thds'] / counts['cpus']), + gpus_per_task=counts['gpus'], + ) + ) + + return jobspecs + + def write_jobspec(self, args, jobspecs): + for idx, jobspec in enumerate(jobspecs): + with open(os.path.join(args.jobspec_dir, "js_{0:}.json".format(idx)), 'w') as f: + f.write(jobspec) + + def main(self, args): + jids = self.submit(args) + + for jid in jids: + print(jid) + +def main(): + parser = argparse.ArgumentParser(prog="jobspec") + subparsers = parser.add_subparsers( + title="supported subcommands", description="", dest="subcommand" + ) + subparsers.required = True + + # run + run = RunCmd() + jobspec_run_parser_sub = subparsers.add_parser( + "run", + parents=[run.get_parser()], + help="generate jobspec for multiple jobs using mpibind", + ) + jobspec_run_parser_sub.set_defaults(func=run.main) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/config/am_check_pymod.m4 b/config/am_check_pymod.m4 new file mode 100644 index 0000000..dd9944e --- /dev/null +++ b/config/am_check_pymod.m4 @@ -0,0 +1,42 @@ +# TODO: LICENSE FROM FLUX, note this is also alvailable in GNOME + +dnl AM_CHECK_PYMOD(MODNAME [,SYMBOL [,ACTION-IF-FOUND [,ACTION-IF-NOT-FOUND]]]) +dnl Check if a module containing a given symbol is visible to python. +AC_DEFUN([AM_CHECK_PYMOD], +[AC_REQUIRE([AM_PATH_PYTHON]) +py_mod_var=`echo "$1_$2" | sed 'y%<>= "./+-(),'"'"'%______p______%'` +AC_MSG_CHECKING(for ifelse([$2],[],,[$2 in ])python module $1) +AC_CACHE_VAL(py_cv_mod_$py_mod_var, [ +ifelse([$2],[], [prog=" +import sys +try: + import $1 +except ImportError: + sys.exit(1) +except: + sys.exit(0) +sys.exit(0)"], [prog=" +import sys +from distutils.version import LooseVersion, StrictVersion +import $1 +if not $2: + sys.exit(1) +"]) +if $PYTHON -c "$prog" 1>&AC_FD_CC 2>&AC_FD_CC + then + eval "py_cv_mod_$py_mod_var=yes" + else + eval "py_cv_mod_$py_mod_var=no" + fi +]) +py_val=`eval "echo \`echo '$py_cv_mod_'$py_mod_var\`"` +if test "x$py_val" != xno; then + AC_MSG_RESULT(yes) + ifelse([$3], [],, [$3 +])dnl +else + AC_MSG_RESULT(no) + ifelse([$4], [],, [$4 +])dnl +fi +]) diff --git a/configure.ac b/configure.ac index 7533309..66f7d44 100644 --- a/configure.ac +++ b/configure.ac @@ -28,6 +28,93 @@ LT_INIT AC_PROG_CC_C99 AC_PROG_LN_S +AC_PROG_MAKE_SET +AC_PROG_AWK + +# Constants to substitute +HWLOC_LIBS=$(pkg-config --libs hwloc) +HWLOC_CFLAGS=$(pkg-config --cflags hwloc) +HWLOC_INCLUDEDIR=$(pkg-config --variable=includedir hwloc) +AC_SUBST([HWLOC_LIBS]) +AC_SUBST([HWLOC_CFLAGS]) +AC_SUBST([HWLOC_INCLUDEDIR]) + +TOPOLOGY_DIR = ./topo-xml +AC_SUBST([TOPOLOGY_DIR]) + +# Checks for header files. +AC_CHECK_HEADERS([stdlib.h string.h sys/param.h unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_C_RESTRICT +AC_TYPE_SIZE_T + +# Checks for library functions. +AC_FUNC_FORK +AC_FUNC_MALLOC +AC_FUNC_MMAP +AC_CHECK_FUNCS([regcomp strchr strerror]) + +################ Python Configuration ################ + +# Set python_version to 3 if no user requested version +if test "X$PYTHON_VERSION" = "X" ; then + if test "X$PYTHON" = "X" ; then + PYTHON_VERSION=3 + fi +fi + +# Do not let AX_PYTHON_DEVEL set PYTHON_SITE_PKG +# AX_PYTHON_DEVEL supplies the PYTHON_VERSION variable +# https://www.gnu.org/software/autoconf-archive/ax_python_devel.html +saved_PYTHON_SITE_PKG=$PYTHON_SITE_PKG +AX_PYTHON_DEVEL([>='3.6']) +PYTHON_SITE_PKG=$saved_PYTHON_SITE_PKG + +# Verify that some version of python was discovered +AM_PATH_PYTHON([$ac_python_version]) +if test "X$PYTHON" = "X"; then + AC_MSG_ERROR([could not find python]) +fi +if test "X$PYTHON" = "X"; then + AC_MSG_ERROR([could not find python]) +fi + +# Verify that the discovered python has the required modules +AM_CHECK_PYMOD(cffi, + [StrictVersion(cffi.__version__) >= StrictVersion('1.1.0')], + , + [AC_MSG_ERROR([could not find python module cffi, version 1.1+ required])] + ) + +AM_CHECK_PYMOD(wheel, + [StrictVersion(wheel.__version__) >= StrictVersion('0.0.0')], + , + [AC_MSG_ERROR([could not find python module wheel, any version should do])] + ) + +# Set up locations for python bindings +AS_VAR_SET(mpibindpydir, $pyexecdir/mpibind) +AC_SUBST(mpibindpydir) +AS_VAR_SET(mpibindpysodir, $pyexecdir/_mpibind) +AC_SUBST(mpibindpysodir) +AC_SUBST(PYTHON_LIBRARY, lib${ac_python_library}.so) + +AS_VAR_SET(mpibindlibdir, $libdir/mpibind) +AC_SUBST(mpibindlibdir) +AS_VAR_SET(mpibindpylinkdir, $mpibindlibdir/python$PYTHON_VERSION) +AC_SUBST(mpibindpylinkdir) + +AS_VAR_SET(mpibindbindingincludedir, $includedir/mpibind/_binding) +AC_SUBST(mpibindbindingincludedir) + +AC_SUBST(PYTHON) + +################ end Python Configuration ################ + +# checks for Modules + +PKG_CHECK_MODULES([HWLOC], [hwloc >= 2.0.1], [], []) # hwloc PKG_CHECK_MODULES([HWLOC], [hwloc >= 2.1]) @@ -81,6 +168,10 @@ AC_CONFIG_FILES([ test-suite/Makefile etc/Makefile etc/mpibind.pc + bindings/Makefile + bindings/python/Makefile + bindings/python/_mpibind/Makefile + bindings/python/mpibind/Makefile ]) AC_OUTPUT diff --git a/src/mpibind.c b/src/mpibind.c index 881adec..78ff54b 100644 --- a/src/mpibind.c +++ b/src/mpibind.c @@ -1502,6 +1502,31 @@ void mpibind_print_mapping(mpibind_t *handle) } } +/* + * Print the mapping for each task. + */ +int mpibind_get_mapping_string(mpibind_t *handle, char *mapping_buffer, size_t buffer_size) +{ + char str1[LONG_STR_SIZE], str2[LONG_STR_SIZE]; + + int i; + int written; + for (i=0; intasks; i++) { + int to_write = buffer_size - strlen(mapping_buffer); + hwloc_bitmap_list_snprintf(str1, sizeof(str1), handle->cpus[i]); + hwloc_bitmap_list_snprintf(str2, sizeof(str2), handle->gpus[i]); + written = snprintf(mapping_buffer + strlen(mapping_buffer), + to_write, + "mpibind: task %2d thds %2d gpus %3s cpus %s\n", + i, handle->nthreads[i], str2, str1); + + if (written < 0 || written > to_write){ + return -1; + } + } + + return 0; +} /* * Environment variables that need to be exported by the runtime. diff --git a/src/mpibind.h b/src/mpibind.h index 85ede8c..92016c3 100644 --- a/src/mpibind.h +++ b/src/mpibind.h @@ -160,6 +160,10 @@ extern "C" { * Print the mapping for each task. */ void mpibind_print_mapping(mpibind_t *handle); + /* + * Get the mapping for each task + */ + int mpibind_get_mapping_string(mpibind_t *handle, char *mapping_buffer, size_t buffer_size); /* * Environment variables that need to be exported by the runtime. * CUDA_VISIBLE_DEVICES --comma separated diff --git a/test-suite/Makefile.am b/test-suite/Makefile.am index 9f39f97..755691a 100644 --- a/test-suite/Makefile.am +++ b/test-suite/Makefile.am @@ -2,18 +2,20 @@ AM_CPPFLAGS = -Wall -Werror -I$(top_srcdir)/src $(HWLOC_CFLAGS) $(TAP_CFLAGS) AM_LDFLAGS = -rpath $(TAP_LIBDIR) LDADD = $(top_srcdir)/src/libmpibind.la $(TAP_LIBS) $(HWLOC_LIBS) -TESTS = \ - error.t \ - environment.t \ - coral_lassen.t \ - epyc_corona.t \ - coral_ea.t \ - cts1_quartz.t +TEST_EXTENSIONS = .t .py -TEST_EXTENSIONS = .t +# C Tests T_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \ - $(top_srcdir)/config/tap-driver.sh + $(top_srcdir)/config/tap-driver.sh + +PROGS = \ + error.t \ + environment.t \ + coral_lassen.t \ + epyc_corona.t \ + coral_ea.t \ + cts1_quartz.t coral_lassen_t_SOURCES = coral-lassen.c test_utils.c test_utils.h epyc_corona_t_SOURCES = epyc-corona.c test_utils.c test_utils.h @@ -23,5 +25,25 @@ error_t_SOURCES = error.c test_utils.c test_utils.h environment_t_SOURCES = environment.c test_utils.c test_utils.h if HAVE_LIBTAP -check_PROGRAMS = $(TESTS) -endif \ No newline at end of file +check_PROGRAMS = $(PROGS) +endif + +# Python Tests +PY_LOG_DRIVER = $(PYTHON) $(top_srcdir)/config/tap-driver.py + +PYSCRIPTS = \ + python/wrapper.py + +AM_TESTS_ENVIRONMENT = \ + export PYTHONPATH="$(abs_top_builddir)/bindings/python:$(abs_top_builddir)/test-suite/python::$$PYTHONPATH";\ + export PYTHON="${PYTHON}";\ + cp $(top_srcdir)/test-suite/tap-driver.py $(top_srcdir)/config/tap-driver.py; + +# Run both Python and C Tests +TESTS = \ + $(PROGS) + +# $(PYSCRIPTS) + +clean-local: + rm -rf trash-directory.* test-results .prove *.broker.log */*.broker.log *.output python/__pycache__ \ No newline at end of file diff --git a/test-suite/python/.gitignore b/test-suite/python/.gitignore new file mode 100644 index 0000000..0e4cbba --- /dev/null +++ b/test-suite/python/.gitignore @@ -0,0 +1,4 @@ +*.pyc +/build +/dist +/*.egg-info diff --git a/test-suite/python/COPYING b/test-suite/python/COPYING new file mode 100644 index 0000000..1b24840 --- /dev/null +++ b/test-suite/python/COPYING @@ -0,0 +1,18 @@ +Copyright (c) 2015 Remko Tronçon (https://el-tramo.be) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/test-suite/python/pycotap/__init__.py b/test-suite/python/pycotap/__init__.py new file mode 100644 index 0000000..ca41dba --- /dev/null +++ b/test-suite/python/pycotap/__init__.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# coding=utf-8 + +# Copyright (c) 2015 Remko Tronçon (https://el-tramo.be) +# Released under the MIT license +# See COPYING for details + + +import unittest +import sys +import base64 + +if sys.hexversion >= 0x03000000: + from io import StringIO +else: + from StringIO import StringIO + +# Log modes +class LogMode(object): + LogToError, LogToDiagnostics, LogToYAML, LogToAttachment = range(4) + + +class TAPTestResult(unittest.TestResult): + def __init__(self, output_stream, error_stream, message_log, test_output_log): + super(TAPTestResult, self).__init__(self, output_stream) + self.output_stream = output_stream + self.error_stream = error_stream + self.orig_stdout = None + self.orig_stderr = None + self.message = error_stream + self.test_output = None + self.message_log = message_log + self.test_output_log = test_output_log + self.output_stream.write("TAP version 13\n") + + def print_raw(self, text): + self.output_stream.write(text) + self.output_stream.flush() + + def print_result(self, result, test, directive=None): + self.output_stream.write("%s %d %s" % (result, self.testsRun, test.id())) + if directive: + self.output_stream.write(" # " + directive) + self.output_stream.write("\n") + self.output_stream.flush() + + def ok(self, test, directive=None): + self.print_result("ok", test, directive) + + def not_ok(self, test): + self.print_result("not ok", test) + + def startTest(self, test): + self.orig_stdout = sys.stdout + self.orig_stderr = sys.stderr + if self.message_log == LogMode.LogToError: + self.message = self.error_stream + else: + self.message = StringIO() + if self.test_output_log == LogMode.LogToError: + self.test_output = self.error_stream + else: + self.test_output = StringIO() + + if self.message_log == self.test_output_log: + self.test_output = self.message + + sys.stdout = sys.stderr = self.test_output + super(TAPTestResult, self).startTest(test) + + def stopTest(self, test): + super(TAPTestResult, self).stopTest(test) + sys.stdout = self.orig_stdout + sys.stderr = self.orig_stderr + if self.message_log == self.test_output_log: + logs = [(self.message_log, self.message, "output")] + else: + logs = [ + (self.test_output_log, self.test_output, "test_output"), + (self.message_log, self.message, "message"), + ] + for log_mode, log, log_name in logs: + if log_mode != LogMode.LogToError: + output = log.getvalue() + if len(output): + if log_mode == LogMode.LogToYAML: + self.print_raw(" ---\n") + self.print_raw(" " + log_name + ": |\n") + self.print_raw( + " " + output.rstrip().replace("\n", "\n ") + "\n" + ) + self.print_raw(" ...\n") + elif log_mode == LogMode.LogToAttachment: + self.print_raw(" ---\n") + self.print_raw(" " + log_name + ":\n") + self.print_raw(" File-Name: " + log_name + ".txt\n") + self.print_raw(" File-Type: text/plain\n") + self.print_raw( + " File-Content: " + base64.b64encode(output) + "\n" + ) + self.print_raw(" ...\n") + else: + self.print_raw( + "# " + output.rstrip().replace("\n", "\n# ") + "\n" + ) + + def addSuccess(self, test): + super(TAPTestResult, self).addSuccess(test) + self.ok(test) + + def addError(self, test, err): + super(TAPTestResult, self).addError(test, err) + self.message.write(self.errors[-1][1] + "\n") + self.not_ok(test) + + def addFailure(self, test, err): + super(TAPTestResult, self).addFailure(test, err) + self.message.write(self.failures[-1][1] + "\n") + self.not_ok(test) + + def addSkip(self, test, reason): + super(TAPTestResult, self).addSkip(test, reason) + self.ok(test, "SKIP " + reason) + + def addExpectedFailure(self, test, err): + super(TAPTestResult, self).addExpectedFailure(test, err) + self.message.write(self.expectedFailures[-1][1] + "\n") + self.ok(test) + + def addUnexpectedSuccess(self, test): + super(TAPTestResult, self).addUnexpectedSuccess(self, test) + self.not_ok(test) + + def printErrors(self): + self.print_raw("1..%d\n" % self.testsRun) + + +class TAPTestRunner(unittest.TextTestRunner): + def __init__( + self, + message_log=LogMode.LogToYAML, + test_output_log=LogMode.LogToDiagnostics, + output_stream=sys.stdout, + error_stream=sys.stderr, + ): + self.output_stream = output_stream + self.error_stream = error_stream + self.message_log = message_log + self.test_output_log = test_output_log + + def run(self, test): + result = TAPTestResult( + self.output_stream, + self.error_stream, + self.message_log, + self.test_output_log, + ) + test(result) + result.printErrors() + + return result diff --git a/test-suite/python/wrapper.py b/test-suite/python/wrapper.py new file mode 100644 index 0000000..e6220a6 --- /dev/null +++ b/test-suite/python/wrapper.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import unittest + +from mpibind.wrapper import MpibindWrapper + +pympibind = MpibindWrapper() + +coral_ea_topo = '../../topo-xml/coral-ea-hwloc1.xml' +coral_ea_expected = '../expected/expected.coral-ea' + +class WrapperTests(unittest.TestCase): + def test_coral_ea_1(self): + +if __name__ == "__main__": + from pycotap import TAPTestRunner + unittest.main(testRunner=TAPTestRunner) \ No newline at end of file diff --git a/test-suite/tap-driver-new.py b/test-suite/tap-driver-new.py new file mode 100644 index 0000000..672cfa5 --- /dev/null +++ b/test-suite/tap-driver-new.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +import sys +import os +import argparse + +def get_tap_driver(driver_path): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), "tap-driver.sh") + +def get_python_executable(): + return sys.executable + +def split_arguments(args): + args_split_point = args.index('--') + return args[:args_split_point], args[args_split_point+1:] + +def build_command(driver_path, python_executable, args): + driver_args, test_path = split_arguments(args) + return [driver_path] + driver_args + ['--', python_executable] + test_path + +if __name__ == "__main__": + driver_path = get_tap_driver(sys.argv[1]) + args = sys.argv[2:] + + full_command = build_command(driver_path, get_python_executable(), args) + #print(full_command) + os.execv(driver_path, full_command) \ No newline at end of file diff --git a/test-suite/tap-driver.py b/test-suite/tap-driver.py new file mode 100644 index 0000000..aa285ed --- /dev/null +++ b/test-suite/tap-driver.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +import sys +import os +from os import path + +def main(): + arguments = sys.argv[1:] # 0 is me + try: + args_split_point = arguments.index('--') + driver_args = arguments[:args_split_point] + test_command = arguments[args_split_point+1:] + except ValueError: + for idx, value in enumerate(arguments): + if not value.startswith('--'): + driver_args = arguments[:idx] + test_command = arguments[idx:] + break + + driver = path.join(path.dirname(path.realpath(__file__)), "tap-driver.sh") + full_command = [driver] + driver_args + ["--", sys.executable] + test_command + os.execv(driver, full_command) + +if __name__ == "__main__": + main() diff --git a/test-suite/test_utils.h b/test-suite/test_utils.h index fc4052e..90bb391 100644 --- a/test-suite/test_utils.h +++ b/test-suite/test_utils.h @@ -3,8 +3,8 @@ #include #include #include +#include #include "mpibind.h" -#include "tap.h" /** * The number of tests present in the test_suite.