upload(wasm): allow uploading workflows defined inside containers (#88)

Author: csegarragonz

README.md

@@ -8,7 +8,7 @@ running [Faasm](https://github.com/faasm/faasm) cluster.
 To install `faasmctl` you need a working `pip` (virtual-)environment. Then:
 
 ```bash
-pip install faasmctl==0.44.0
+pip install faasmctl==0.45.0
 ```
 
 ## Usage

faasmctl/tasks/upload.py

@@ -1,5 +1,7 @@
 from faasmctl.util.upload import upload_file, upload_wasm
 from invoke import task
+from os.path import join
+from subprocess import run
 
 
 @task(default=True)
@@ -17,6 +19,37 @@ def wasm(ctx, user, func, wasm_file, ini_file=None):
     upload_wasm(user, func, wasm_file, ini_file)
 
 
+@task
+def workflow(ctx, wflow, path, ini_file=None):
+    """
+    Upload all WASM files corresponding to a workflow (from --path)
+
+    Path indicates a directory where each function in the workflow is in a
+    different sub-directory, in a single WASM file. Uploading a workflow is
+    equivalent to uplodaing a `user` in the traditional Faasm sense. Note that
+    you may prefix the path with a container name, followed by a colon, and
+    then the path, to copy the file from a path inside a container.
+    """
+    wasm_in_ctr = path.rfind(":") != -1
+    if wasm_in_ctr:
+        ctr_image = path[: path.rfind(":")]
+        in_ctr_path = path[path.rfind(":") + 1 :]
+        docker_cmd = "docker run --rm -it {} ls --format=commas {}".format(
+            ctr_image, in_ctr_path
+        )
+        funcs = (
+            run(docker_cmd, shell=True, capture_output=True)
+            .stdout.decode("utf-8")
+            .strip()
+            .split(", ")
+        )
+
+        for func in funcs:
+            upload_wasm(wflow, func, join(path, func, "function.wasm"))
+    else:
+        raise RuntimeError("Not implemented!")
+
+
 @task()
 def file(ctx, host_path, faasm_path, ini_file=None):
     """

faasmctl/util/compose.py

@@ -11,6 +11,7 @@
 from subprocess import run
 from time import sleep
 
+DEFAULT_FAASM_CAPTURE_STDOUT = "on"
 DEFAULT_FAASM_OVERRIDE_CPU_COUNT = "8"
 
 
@@ -114,6 +115,10 @@ def get_compose_env_vars(faasm_checkout, mount_source, ini_file=None):
     if "FAASM_OVERRIDE_CPU_COUNT" in environ:
         env["FAASM_OVERRIDE_CPU_COUNT"] = environ["FAASM_OVERRIDE_CPU_COUNT"]
 
+    env["FAASM_CAPTURE_STDOUT"] = DEFAULT_FAASM_CAPTURE_STDOUT
+    if "FAASM_CAPTURE_STDOUT" in environ:
+        env["FAASM_CAPTURE_STDOUT"] = environ["FAASM_CAPTURE_STDOUT"]
+
     if "FAASM_CLI_IMAGE" in environ:
         env["FAASM_CLI_IMAGE"] = environ["FAASM_CLI_IMAGE"]
 

faasmctl/util/upload.py

@@ -4,6 +4,7 @@
 )
 from faasmctl.util.docker import in_docker
 from requests import put
+from subprocess import run
 
 
 def upload_wasm(user, func, wasm_file, ini_file=None):
@@ -13,11 +14,38 @@ def upload_wasm(user, func, wasm_file, ini_file=None):
     if not ini_file:
         ini_file = get_faasm_ini_file()
 
+    # Work out if WASM file is a host path, or a path in a container
+    wasm_in_ctr = wasm_file.rfind(":") != -1
+    if wasm_in_ctr:
+        tmp_ctr_name = "wasm-ctr"
+
+        def stop_ctr():
+            run(f"docker rm -f {tmp_ctr_name}", shell=True, capture_output=True)
+
+        ctr_image = wasm_file[: wasm_file.rfind(":")]
+        in_ctr_path = wasm_file[wasm_file.rfind(":") + 1 :]
+        docker_cmd = "docker run -d --name {} {} bash".format(tmp_ctr_name, ctr_image)
+        run(docker_cmd, shell=True, capture_output=True)
+
+        tmp_wasm_file = "/tmp/wasm-ctr-func.wasm"
+        docker_cmd = "docker cp {}:{} {}".format(
+            tmp_ctr_name, in_ctr_path, tmp_wasm_file
+        )
+        try:
+            run(docker_cmd, shell=True, capture_output=True)
+        except Exception as e:
+            print("Caught exception copying: {}".format(e))
+
+        stop_ctr()
+
+        wasm_file = tmp_wasm_file
+
     host, port = get_faasm_upload_host_port(ini_file, in_docker())
     url = "http://{}:{}/f/{}/{}".format(host, port, user, func)
 
     response = put(url, data=open(wasm_file, "rb"))
-    print("Response ({}): {}".format(response.status_code, response.text))
+    if response.status_code != 200:
+        raise RuntimeError(f"Error uploading WASM: {response.text}")
 
 
 def upload_python(func, python_file, ini_file=None):

faasmctl/util/version.py

@@ -1,4 +1,4 @@
-FAASMCTL_VERSION = "0.44.0"
+FAASMCTL_VERSION = "0.45.0"
 
 
 def get_version():

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "faasmctl"
-version = "0.44.0"
+version = "0.45.0"
 authors = [
   { name="Faasm Team", email="[email protected]" },
 ]