WIP: Mpibind jobspec

.gitignore

@@ -31,7 +31,8 @@ libltdl/
 /config/lt~obsolete.m4
 /config/ar-lib
 /config/tap-driver.sh
-
+/config/tap-driver.py
+/config/py-compile
 # docs intermediate files
 /doc/man*/*.xml
 /doc/_build

Makefile.am

@@ -1,4 +1,5 @@
+.NOTPARALLEL:
 
-SUBDIRS = src test-suite etc
+SUBDIRS = src test-suite etc bindings
 
-ACLOCAL_AMFLAGS = -I config
+ACLOCAL_AMFLAGS = -I config

bindings/Makefile.am

@@ -0,0 +1 @@
+SUBDIRS = python

bindings/README.md

@@ -0,0 +1,17 @@
+## Bindings for mpibind
+
+### Overview
+
+In order to improve its usability, we offer bindings for the mpibind library in
+languagues beyond C. Currently the only other language supported is Python.
+
+### License
+
+*mpibind* is distributed under the terms of the MIT license. All new
+contributions must be made under this license. 
+
+See [LICENSE](LICENSE) and [NOTICE](NOTICE) for details.
+
+SPDX-License-Identifier: MIT.
+
+LLNL-CODE-812647.

bindings/python/Makefile.am

@@ -0,0 +1,19 @@
+.NOTPARALLEL:
+
+SUBDIRS= _mpibind mpibind
+
+AM_CPPFLAGS = \
+	-Wall -Werror -Wno-missing-field-initializers \
+	-I$(top_srcdir) -I$(top_srcdir)/src/ -L$(top_srcdir)/src/.libs \
+	$(PYTHON_CPPFLAGS)
+
+all-local:
+	(cd $(srcdir); CFLAGS="$(AM_CPPFLAGS)" \
+		$(PYTHON) setup.py bdist_wheel)
+
+clean-local:
+	-rm -f *.pyc *.pyo
+	-rm -rf __pycache__
+	-rm -rf mpibind.egg-info
+	-rm -rf build
+	-rm -rf dist

bindings/python/README.md

@@ -0,0 +1,25 @@
+## Python Bindings for mpibind
+
+### Overview
+
+You can interact with mpibind through your python scripts:
+
+```python
+# running on syrah login node
+from mpibind import MpibindWrapper
+wrapper = MpibindWrapper()
+mapping = wrapper.get_mapping(ntasks=4)
+```
+
+### Installing
+
+### License
+
+*mpibind* is distributed under the terms of the MIT license. All new
+contributions must be made under this license. 
+
+See [LICENSE](LICENSE) and [NOTICE](NOTICE) for details.
+
+SPDX-License-Identifier: MIT.
+
+LLNL-CODE-812647.

bindings/python/_mpibind/Makefile.am

@@ -0,0 +1,54 @@
+AM_CPPFLAGS = \
+	-Wall -Werror -Wno-missing-field-initializers \
+	-I$(top_srcdir) -I$(top_srcdir)/src/ \
+	$(PYTHON_CPPFLAGS)
+
+AM_LDFLAGS = \
+	-avoid-version -module -Wl,-rpath,$(PYTHON_PREFIX)/lib
+
+_pympibind.c: $(srcdir)/build.py _mpibind_preproc.h
+	$(PYTHON) $<
+
+# Flux LICENSE
+_mpibind_preproc.h: _mpibind_clean.h
+	$(CC) -E  '-D__attribute__(...)=' _mpibind_clean.h\
+	  | sed -e '/^# [0-9]*.*/d' > $@
+
+_mpibind_clean.h: Makefile
+	$(PYTHON) $(srcdir)/clean_header.py \
+	$(top_builddir)/src/mpibind.h \
+	_mpibind_clean.h 
+
+BUILT_SOURCES= _pympibind.c _mpibind_preproc.h
+mpibindpyso_LTLIBRARIES = _pympibind.la
+mpibindpyso_PYTHON = __init__.py
+
+nodist_mpibindbindinginclude_HEADERS = _mpibind_preproc.h
+nodist__pympibind_la_SOURCES = _pympibind.c
+
+common_libs = $(top_builddir)/src/libmpibind.la \
+		  $(PYTHON_LDFLAGS)
+
+_pympibind_la_LIBADD = $(common_libs)
+
+EXTRA_DIST=build.py make_clean_header.py
+
+clean-local:
+	-rm -f *.c *.so *.pyc *.pyo *_clean.h *_preproc.h
+	-rm -rf __pycache__
+	-rm -rf .libs
+
+# creates a symbolic link _pympibind.so to 
+# _pympibind.so in _mpibind/.libs so that it can be imported in mpibind/wrapper
+# Flux LICENSE
+
+.PHONY: lib-copy
+
+lib-copy: ${mpibindpyso_PYTHON} ${mpibindpyso_LTLIBRARIES}
+	-echo Copying libraries to where they can be used by python in-tree
+	for LIB in ${mpibindpyso_LTLIBRARIES:la=so} ; do \
+		test -e .libs/$$LIB && \
+		$(LN_S) .libs/$$LIB ./ || true; \
+	done
+
+all-local: lib-copy

bindings/python/_mpibind/build.py

@@ -0,0 +1,36 @@
+from cffi import FFI
+from os import path
+
+ffibuilder = FFI()
+
+# add some of the opaque hwloc objects
+cdefs = """
+typedef struct { ...; } hwloc_topology_t;
+typedef struct { ...; } hwloc_bitmap_t;
+"""
+
+# this file is used from two different places during build
+# and then during distribution
+prepared_headers = "_mpibind_preproc.h"
+if not path.exists(prepared_headers):
+    prepared_headers = "_mpibind/_mpibind_preproc.h"
+
+# add cleaned header file to the definitions we expose to python
+with open(prepared_headers) as h:
+    cdefs = cdefs + h.read()
+
+# set the defintions we expose to python
+ffibuilder.cdef(cdefs)
+
+# indicate that the library makes the definitions we set
+ffibuilder.set_source(
+    "_mpibind._pympibind",
+    """
+    #include <src/mpibind.h>
+    """,
+    libraries=["mpibind"],
+)
+
+#emit c code and let autotools build _pympibind.so extension module
+if __name__ == "__main__":
+    ffibuilder.emit_c_code("_pympibind.c")

bindings/python/_mpibind/clean_header.py

@@ -0,0 +1,56 @@
+import os
+import re
+import struct
+
+import argparse
+
+platform_c_maxint = 2 ** (struct.Struct('i').size * 8 - 1) - 1
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("header", help="header file to parse", type=str)
+    parser.add_argument("output", help="destination of clean header", type=str)
+
+    return parser.parse_args()
+
+def apply_replacement_rules(line):
+    rules = {
+        "__restrict": "restrict",
+        "__inline__": "inline",
+        "INT_MAX": str(platform_c_maxint),
+    }
+
+    for k,v in rules.items():
+        line = line.replace(k, v)
+
+    if ("inline" in line):
+        print(line)
+    return line
+
+def make_enum_literal(line):
+    return re.sub(r'(\d)(UL)?<<(\d)',lambda x: str(int(x.group(1))<<int(x.group(3))),line)
+
+def remove_external_headers(line):
+    return re.sub(r'#\s*include\s+<.+>', '', line)
+
+def clean_header(header):
+    cleaned = ""
+    with open(header, "r") as f:
+        for line in f.readlines():
+            line = apply_replacement_rules(line)
+            line = make_enum_literal(line)
+            line = remove_external_headers(line)
+            cleaned += line
+
+    return cleaned
+
+def output_header(output_file, cleaned):
+    with open(output_file, "w") as f:
+        f.write(cleaned)
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    abs_header = os.path.abspath(args.header)
+    abs_output = os.path.abspath(args.output)
+    output_header(abs_output, clean_header(abs_header))

bindings/python/mpibind/Makefile.am

@@ -0,0 +1,7 @@
+mpibindpy_PYTHON=\
+	      __init__.py\
+		  wrapper.py
+
+clean-local:
+	-rm -f *.pyc *.pyo
+	-rm -rf __pycache__