libfaasm: add support for general S3 API (#126)

* libfaasm: add support for general s3 API

* s3: add function to get number of buckets

* s3: add remaining basic functions

* nits: run clang-format

* gh: bump minor code version

* libfaasm: remove un-implemented symbols

* s3: temporarily commit the workflow functions

* wc: more functions

* s3: untrack wc functions

* s3: build functions as part of gha

* func(demo): update chain_output function for new signature

* func: separate threaded/non-threaded targets

Author: csegarragonz

.env

@@ -1,3 +1,3 @@
-SYSROOT_VERSION=0.5.0
-SYSROOT_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.5.0
+SYSROOT_VERSION=0.6.0
+SYSROOT_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.6.0
 COMPOSE_PROJECT_NAME=cpp-dev

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm.azurecr.io/cpp-sysroot:0.5.0
+      image: faasm.azurecr.io/cpp-sysroot:0.6.0
     steps:
       # --- Update code ---
       - name: "Checkout code"

VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

func/CMakeLists.txt

@@ -74,4 +74,5 @@ add_subdirectory(dynlink)
 add_subdirectory(errors)
 add_subdirectory(mpi)
 add_subdirectory(omp)
+add_subdirectory(s3)
 add_subdirectory(threads)

func/demo/chain_output.cpp

@@ -26,28 +26,37 @@ int main(int argc, char* argv[])
     unsigned int callIdB = faasmChain(otherB, nullptr, 0);
 
     std::string expectedA = "expected A";
-    std::string actualA;
-    actualA.reserve(expectedA.size());
 
     std::string expectedB = "longer expected B";
-    std::string actualB;
-    actualB.reserve(expectedB.size());
 
+    std::string actualA;
+    char* actualABuf;
+    int actualABufSize;
     unsigned int resA =
-      faasmAwaitCallOutput(callIdA, actualA.c_str(), actualA.size());
+      faasmAwaitCallOutput(callIdA, &actualABuf, &actualABufSize);
+    actualA.assign(actualABuf, actualABuf + actualABufSize);
+
+    std::string actualB;
+    char* actualBBuf;
+    int actualBBufSize;
     unsigned int resB =
-      faasmAwaitCallOutput(callIdB, actualB.c_str(), actualB.size());
+      faasmAwaitCallOutput(callIdB, &actualBBuf, &actualBBufSize);
+    actualB.assign(actualBBuf, actualBBuf + actualBBufSize);
 
     if (resA != 0 || resB != 0) {
         printf("One or more chained calls failed: %i %i\n", resA, resB);
         return 1;
     }
 
     if (actualA != expectedA) {
+        printf(
+          "Output mismatch: %s != %s\n", actualA.c_str(), expectedA.c_str());
         return 1;
     }
 
     if (actualB != expectedB) {
+        printf(
+          "Output mismatch: %s != %s\n", actualB.c_str(), expectedB.c_str());
         return 1;
     }
 

func/s3/CMakeLists.txt

@@ -0,0 +1,16 @@
+set(FAASM_USER s3)
+
+function(s3_func exec_name dir_path)
+    faasm_func(${exec_name} ${dir_path})
+    set(ALL_DEMO_FUNCS ${ALL_DEMO_FUNCS} ${exec_name} PARENT_SCOPE)
+endfunction(s3_func)
+
+s3_func(get_num_buckets get_num_buckets.cpp)
+s3_func(list_buckets list_buckets.cpp)
+s3_func(get_num_keys get_num_keys.cpp)
+s3_func(list_keys list_keys.cpp)
+s3_func(add_key_bytes add_key_bytes.cpp)
+s3_func(get_key_bytes get_key_bytes.cpp)
+
+# Custom target to group all the demo functions
+add_custom_target(s3_all_funcs DEPENDS ${ALL_DEMO_FUNCS})

func/s3/add_key_bytes.cpp

@@ -0,0 +1,30 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+
+int main(int argc, char* argv[])
+{
+    // Get bucket and key from command line
+    if (argc != 3) {
+        printf("error: must invoke function with two arguments: bucketName "
+               "keyName\n");
+        return 1;
+    }
+
+    char* bucketName = argv[1];
+    char* keyName = argv[2];
+
+    // Get the bytes to add as input
+    int inputSize = faasmGetInputSize();
+    uint8_t keyBytes[inputSize];
+    faasmGetInput(keyBytes, inputSize);
+
+    int ret =
+      __faasm_s3_add_key_bytes(bucketName, keyName, (void*)keyBytes, inputSize);
+
+    return ret;
+}

func/s3/get_key_bytes.cpp

@@ -0,0 +1,30 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+
+int main(int argc, char* argv[])
+{
+    // Get bucket and key from command line
+    if (argc != 3) {
+        printf("error: must invoke function with two arguments: bucketName "
+               "keyName\n");
+        return 1;
+    }
+
+    char* bucketName = argv[1];
+    char* keyName = argv[2];
+
+    uint8_t* keyBytes;
+    int keyBytesLen;
+
+    int ret =
+      __faasm_s3_get_key_bytes(bucketName, keyName, &keyBytes, &keyBytesLen);
+    printf("Got %s/%s: %s\n", bucketName, keyName, (char*)keyBytes);
+    faasmSetOutput((char*)keyBytes, keyBytesLen);
+
+    return ret;
+}

func/s3/get_num_buckets.cpp

@@ -0,0 +1,20 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+#include <string>
+
+int main(int argc, char* argv[])
+{
+    int numBuckets = __faasm_s3_get_num_buckets();
+
+    printf("Got %i buckets!\n", numBuckets);
+
+    std::string numBucketsStr = std::to_string(numBuckets);
+    faasmSetOutput(numBucketsStr.c_str(), numBucketsStr.size());
+
+    return 0;
+}

func/s3/get_num_keys.cpp

@@ -0,0 +1,25 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+#include <string>
+
+int main(int argc, char* argv[])
+{
+    // Get the bucket name as an input
+    int inputSize = faasmGetInputSize();
+    char bucketName[inputSize];
+    faasmGetInput((uint8_t*)bucketName, inputSize);
+
+    int numKeys = __faasm_s3_get_num_keys(bucketName);
+
+    printf("Bucket %s has %i keys!\n", bucketName, numKeys);
+
+    std::string numKeysStr = std::to_string(numKeys);
+    faasmSetOutput(numKeysStr.c_str(), numKeysStr.size());
+
+    return 0;
+}