s3: add client api to interact with s3 server (#87)

* s3: add preliminary support to interact with s3

* s3: add more functionality and split in two files

Author: csegarragonz

faasmctl/tasks/__init__.py

@@ -9,6 +9,7 @@
 from . import logs
 from . import monitor
 from . import restart
+from . import s3
 from . import scale
 from . import status
 from . import upload
@@ -23,6 +24,7 @@
     logs,
     monitor,
     restart,
+    s3,
     scale,
     status,
     upload,

faasmctl/tasks/s3.py

@@ -0,0 +1,59 @@
+from invoke import task
+from faasmctl.util.s3 import (
+    list_buckets as do_list_buckets,
+    clear_bucket as do_clear_bucket,
+    list_objects as do_list_objects,
+    upload_file as do_upload_file,
+    upload_dir as do_upload_dir,
+    dump_object as do_dump_object,
+)
+
+
+@task
+def list_buckets(ctx):
+    """
+    List available buckets
+    """
+    do_list_buckets()
+
+
+@task
+def clear_bucket(ctx, bucket):
+    """
+    Clear (i.e. remove) bucket
+    """
+    do_clear_bucket(bucket)
+
+
+@task
+def list_objects(ctx, bucket, recursive=False):
+    """
+    List available objects in bucket
+    """
+    do_list_objects(bucket, recursive)
+
+
+@task
+def upload_file(ctx, bucket, host_path, s3_path):
+    """
+    Upload a file to S3
+    """
+    do_upload_file(bucket, host_path, s3_path)
+
+
+@task
+def upload_dir(ctx, bucket, host_path, s3_path):
+    """
+    Upload all the files in a directory to S3
+    """
+    do_upload_dir(bucket, host_path, s3_path)
+
+
+@task
+def dump_object(ctx, bucket, path):
+    """
+    Dump the contents of an object in S3
+    """
+    response = do_dump_object(bucket, path)
+
+    print(response.data)

faasmctl/util/s3.py

@@ -0,0 +1,73 @@
+from faasmctl.util.config import get_faasm_ini_file, get_faasm_ini_value
+from minio import Minio
+from minio.error import S3Error
+from os import listdir
+from os.path import isfile, join
+
+
+def get_minio_client():
+    minio_port = get_faasm_ini_value(get_faasm_ini_file(), "Faasm", "minio_port")
+
+    client = Minio(
+        "localhost:{}".format(minio_port),
+        access_key="minio",
+        secret_key="minio123",
+        secure=False,
+        region="",
+    )
+
+    return client
+
+
+def list_buckets():
+    client = get_minio_client()
+    for bucket in client.list_buckets():
+        print(bucket)
+
+
+def list_objects(bucket, recursive=False):
+    client = get_minio_client()
+    for bucket_key in client.list_objects(bucket, recursive=recursive):
+        print(bucket_key.object_name)
+
+
+def clear_bucket(bucket):
+    client = get_minio_client()
+
+    # Got to make sure the bucket is empty first
+    for bucket_key in client.list_objects(bucket, recursive=True):
+        client.remove_object(bucket, bucket_key.object_name)
+
+    client.remove_bucket(bucket)
+
+
+def upload_file(bucket, host_path, s3_path):
+    client = get_minio_client()
+
+    # Create the bucket if it does not exist
+    found = client.bucket_exists(bucket)
+    if not found:
+        client.make_bucket(bucket)
+
+    # Upload the file, renaming it in the process
+    try:
+        client.fput_object(bucket, s3_path, host_path)
+    except S3Error as ex:
+        print("error: error uploading file to s3: {}".format(ex))
+        raise RuntimeError("error: error uploading file to s3")
+
+
+def upload_dir(bucket, host_path, s3_path):
+    for f in listdir(host_path):
+        host_file_path = join(host_path, f)
+
+        if isfile(host_file_path):
+            s3_file_path = join(s3_path, f)
+            upload_file(bucket, host_file_path, s3_file_path)
+
+
+def dump_object(bucket, path):
+    client = get_minio_client()
+    response = client.get_object(bucket, path)
+
+    return response

pyproject.toml

@@ -16,9 +16,10 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-    "invoke == 2.1.3",
-    "requests == 2.31.0",
-    "protobuf == 4.23.4",
+    "invoke >= 2.1.3",
+    "requests >= 2.31.0",
+    "protobuf >= 4.23.4",
+    "minio >= 7.2.7",
 ]
 
 [project.urls]