Tidy-up and latest CPython (#4)

* Removed libffi imports from this dir

* Latest sysroot

* Version bump

* Typo

* Small syntax fix

* Fixing discrepancies in host interface calls

* Refactor of runtime task

* Add copying lib and includes to cpython build

* Refactoring pyfaasm

* Add faasm library to linking

* Update tasks env

* Removed use of special wasm interface library

* Latest cpython

* Latest cpython

* Removed unused task

* Fix for cpython copy missing dir

* Sysroot version

* Fix tests

Author: Shillaker

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-20.04
     container:
-      image: faasm/cpython:0.0.8
+      image: faasm/cpython:0.0.9
     defaults:
       run:
         working-directory: /code/faasm-cpython
@@ -36,7 +36,7 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: faasm/cpython:0.0.8
+      image: faasm/cpython:0.0.9
     defaults:
       run:
         working-directory: /code/faasm-cpython

Dockerfile

@@ -1,4 +1,4 @@
-FROM faasm/cpp-sysroot:0.0.16
+FROM faasm/cpp-sysroot:0.0.18
 ARG FAASM_CPYTHON_VERSION
 
 RUN apt install -y \
@@ -26,7 +26,6 @@ RUN git clone \
 WORKDIR /code/faasm-cpython
 RUN git submodule update --init
 
-
 # Install pyfaasm natively
 RUN inv pyfaasm.native
 

README.md

@@ -3,7 +3,7 @@
 This build cross-compiles CPython and a number of Python modules to WebAssembly
 for use in [Faasm](https://github.com/faasm/faasm).
 
-It also provides a [small Python librar](pyfaasm/) which uses `ctypes` to
+It also provides a [small Python library](pyfaasm/) which uses `ctypes` to
 support calls to the [Faasm host
 interface](https://github.com/faasm/faasm/blob/master/docs/host_interface.md).
 
@@ -47,7 +47,7 @@ Several of the code changes to CPython and numpy were borrowed from
 This repo gets built as a container, `faasm/cpython`. If you want to release a
 new version, you can:
 
-- Update the version in `VERSION` and `.env`
+- Update the version in `VERSION`, `.github` and `.env`
 - Commit to your branch
 - Check that the normal build works in CI
 - Run `inv git.tag`

VERSION

@@ -1 +1 @@
-0.0.8
+0.0.9

libffi.imports

@@ -7,62 +7,96 @@
 # Faasm host interface:
 # https://github.com/faasm/faasm/blob/master/include/faasm/host_interface.h
 
-HOST_INTERFACE_LIB = "libemulator.so"
-
-SUPPORTING_LIBS = [
+NATIVE_SUPPORTING_LIBS = [
     "libpistache.so",
     "libfaabricmpi.so",
 ]
 
-HOST_INTERFACE_LIB = "libemulator.so"
-PYTHON_LOCAL_CHAINING = bool(os.environ.get("PYTHON_LOCAL_CHAINING"))
-PYTHON_LOCAL_OUTPUT = bool(os.environ.get("PYTHON_LOCAL_OUTPUT"))
+NATIVE_INTERFACE_LIB = "libemulator.so"
+
+env_cache = dict()
 
 input_data = None
 output_data = None
 
 _host_interface = None
 
 
-def _init_host_interface():
-    global _host_interface
+def get_env_bool(var_name):
+    global env_cache
 
-    if _host_interface is None:
-        # Load all supporting libs as globally linkable
-        for lib in SUPPORTING_LIBS:
-            ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
+    if var_name not in env_cache:
+        value = os.environ.get(var_name)
+        env_cache[var_name] = bool(value)
+
+    return env_cache[var_name]
+
+
+def set_env_bool(var_name, value):
+    global env_cache
+    env_cache[var_name] = value
+
+
+def is_wasm():
+    return get_env_bool("PYTHONWASM")
+
+
+def is_local_chaining():
+    return get_env_bool("PYTHON_LOCAL_CHAINING")
 
-        # Load main Faasm host interface lib
-        _host_interface = ctypes.CDLL("libemulator.so")
-        print("Loaded Faasm host interface: {}".format(_host_interface))
+
+def is_local_output():
+    return get_env_bool("PYTHON_LOCAL_OUTPUT")
 
 
 def set_local_chaining(value):
-    global PYTHON_LOCAL_CHAINING
-    PYTHON_LOCAL_CHAINING = value
+    set_env_bool("PYTHON_LOCAL_CHAINING", value)
 
 
 def set_local_input_output(value):
-    global PYTHON_LOCAL_OUTPUT
-    PYTHON_LOCAL_OUTPUT = value
+    set_env_bool("PYTHON_LOCAL_OUTPUT", value)
+
+
+def _init_host_interface():
+    global _host_interface
+
+    if _host_interface is None:
+        # Wasm and native environments are different
+        if is_wasm():
+            # Wasm expects the main application to handle the relevant calls
+            _host_interface = ctypes.CDLL(None)
+        else:
+            # Load all supporting libs as globally linkable
+            for lib in NATIVE_SUPPORTING_LIBS:
+                ctypes.CDLL(lib, mode=ctypes.RTLD_GLOBAL)
+
+            # Load main Faasm host interface lib
+            _host_interface = ctypes.CDLL(NATIVE_INTERFACE_LIB)
+
+
+def get_input_len():
+    _init_host_interface()
+    return _host_interface.__faasm_read_input(None, 0)
 
 
-def read_input():
+def read_input(input_len):
     _init_host_interface()
-    return _host_interface.__faasm_read_input()
+    input_len = int(input_len)
+    buff = ctypes.create_string_buffer(input_len)
+    return _host_interface.__faasm_read_input(buff, input_len)
 
 
 def write_output(output):
-    if PYTHON_LOCAL_OUTPUT:
+    if is_local_output():
         global output_data
         output_data = output
     else:
         _init_host_interface()
-        _host_interface.__faasm_write_output(output)
+        _host_interface.__faasm_write_output(output, len(output))
 
 
 def get_output():
-    if PYTHON_LOCAL_OUTPUT:
+    if is_local_output():
         global output_data
         return output_data
     else:
@@ -129,7 +163,7 @@ def pull_state(key, state_len):
 
 
 def chain(func, input_data):
-    if PYTHON_LOCAL_CHAINING:
+    if is_local_chaining():
         # Run function directly
         func(input_data)
         return 0
@@ -144,7 +178,7 @@ def chain(func, input_data):
 
 
 def await_call(call_id):
-    if PYTHON_LOCAL_CHAINING:
+    if is_local_chaining():
         # Calls are run immediately in local chaining
         return 0
     else:
@@ -156,7 +190,7 @@ def set_emulator_message(message_json):
     _init_host_interface()
 
     message_bytes = bytes(message_json, "utf-8")
-    if PYTHON_LOCAL_OUTPUT:
+    if is_local_output():
         global output_data
         output_data = None
 

pyfaasm/pyfaasm/core.py

@@ -6,7 +6,7 @@
     set_emulator_message,
     get_output,
     write_output,
-    PYTHON_LOCAL_OUTPUT,
+    is_local_output,
 )
 
 
@@ -16,7 +16,7 @@ class TestEmulator(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         # Check what local output already set to
-        cls.original_local_output = PYTHON_LOCAL_OUTPUT
+        cls.original_local_output = is_local_output()
 
     @classmethod
     def tearDownClass(cls):

pyfaasm/test/test_emulator.py

@@ -1,11 +1,12 @@
 import os
+import shutil
 
 from copy import copy
-from tasks.env import PROJ_ROOT, USABLE_CPUS
+from tasks.env import PROJ_ROOT, USABLE_CPUS, FAASM_RUNTIME_ROOT
 from os.path import join, exists
 from subprocess import run
 
-from faasmtools.build import build_config_cmd
+from faasmtools.build import build_config_cmd, WASM_LIB_INSTALL, WASM_SYSROOT
 
 from invoke import task
 
@@ -39,6 +40,17 @@
     }
 )
 
+WASM_INCLUDES_DIR = join(WASM_SYSROOT, "include")
+
+LIB_SRC_DIR = join(INSTALL_DIR, "lib")
+LIB_DEST_DIR = join(FAASM_RUNTIME_ROOT, "lib")
+
+LIBPYTHON_SRC_PATH = join(LIB_SRC_DIR, "libpython3.8.a")
+LIBPYTHON_DEST_PATH = join(WASM_LIB_INSTALL, "libpython3.8.a")
+
+INCLUDE_SRC_DIR = join(INSTALL_DIR, "include", "python3.8")
+INCLUDE_DEST_DIR = join(WASM_INCLUDES_DIR, "python3.8")
+
 # See the CPython docs for more info:
 # - General: https://devguide.python.org/setup/#compile-and-build
 # - Static builds: https://wiki.python.org/moin/BuildStatically
@@ -66,7 +78,7 @@ def build(ctx, clean=False, noconf=False, nobuild=False):
     # relevant in the module builds.
 
     # Link in extra wasi-libc long double support (see wasi-libc docs)
-    link_libs = "-lc-printscan-long-double"
+    link_libs = "-lc-printscan-long-double -lfaasm"
 
     # Configure
     configure_cmd = build_config_cmd(
@@ -100,3 +112,32 @@ def build(ctx, clean=False, noconf=False, nobuild=False):
     # Run specific install tasks (see cpython/Makefile)
     _run_cpython_cmd("commoninstall", ["make", "commoninstall"])
     _run_cpython_cmd("bininstall", ["make", "bininstall"])
+
+    # Prepare destinations
+    os.makedirs(WASM_INCLUDES_DIR, exist_ok=True)
+    os.makedirs(WASM_LIB_INSTALL, exist_ok=True)
+
+    shutil.rmtree(INCLUDE_DEST_DIR, ignore_errors=True)
+
+    print(
+        "Copying libpython from {} to {}".format(
+            LIBPYTHON_SRC_PATH, LIBPYTHON_DEST_PATH
+        )
+    )
+    print(
+        "Copying includes from {} to {}".format(
+            INCLUDE_SRC_DIR, INCLUDE_DEST_DIR
+        )
+    )
+
+    shutil.copy(LIBPYTHON_SRC_PATH, LIBPYTHON_DEST_PATH)
+    shutil.copytree(INCLUDE_SRC_DIR, INCLUDE_DEST_DIR)
+
+    os.makedirs(LIB_DEST_DIR, exist_ok=True)
+
+    print("Copying contents of {} to {}".format(LIB_SRC_DIR, LIB_DEST_DIR))
+    run(
+        "cp -r {}/* {}/".format(LIB_SRC_DIR, LIB_DEST_DIR),
+        shell=True,
+        check=True,
+    )

tasks/cpython.py

@@ -1,11 +1,14 @@
 from multiprocessing import cpu_count
 from os.path import join, dirname, realpath
+from faasmtools.build import FAASM_LOCAL_DIR
 
 PROJ_ROOT = dirname(dirname(realpath(__file__)))
 
 THIRD_PARTY_DIR = join(PROJ_ROOT, "third-party")
 CROSSENV_DIR = join(PROJ_ROOT, "cross_venv", "cross")
 
+FAASM_RUNTIME_ROOT = join(FAASM_LOCAL_DIR, "runtime_root")
+
 USABLE_CPUS = str(int(cpu_count()) - 1)
 VERSION_FILE = join(PROJ_ROOT, "VERSION")