Replace Goofys with mountpoint-s3 for S3 mounting (#417)

* Replace Goofys with mountpoint-s3 for S3 mounting

This update replaces the Goofys S3 filesystem with mountpoint-s3 throughout the codebase, including Dockerfile, shell scripts, and utility functions. The decode_run_json and create_mount_command_list functions now support AWS region and KMS key ARN for encrypted mounts. Tests and documentation have been updated accordingly. Version bumped to 5.6.0.

* Update top.py

* Bump version

Author: alexander-veit

CHANGELOG.rst

@@ -3,6 +3,12 @@
 Change Log
 ==========
 
+6.0.0
+=====
+
+* Replace Goofys with mountpoint-s3 (https://github.com/awslabs/mountpoint-s3)
+
+
 5.5.3
 =====
 

awsf3-docker/Dockerfile

@@ -57,10 +57,22 @@ RUN ARCH="$(dpkg --print-architecture)" && \
     apt install -y ./singularity-ce_${SINGULARITY_VERSION}-focal_${ARCH}.deb && \
     rm singularity-ce_${SINGULARITY_VERSION}-focal_${ARCH}.deb
 
-# goofys
-# RUN wget https://github.com/kahing/goofys/releases/download/v0.24.0/goofys && chmod +x goofys
-RUN ARCH="$(dpkg --print-architecture)" && \
-    wget https://tibanna-dependencies.s3.amazonaws.com/goofys/v0.24.0/${ARCH}/goofys && chmod +x goofys
+# mountpoint-s3
+RUN set -eux; \
+    ARCH="$(dpkg --print-architecture)"; \
+    if [ "$ARCH" = "amd64" ]; then \
+        DEB_URL="https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.deb"; \
+    elif [ "$ARCH" = "arm64" ]; then \
+        DEB_URL="https://s3.amazonaws.com/mountpoint-s3-release/latest/arm64/mount-s3.deb"; \
+    else \
+        echo "Unsupported architecture: $ARCH" >&2; \
+        exit 1; \
+    fi; \
+    wget "$DEB_URL"; \
+    apt-get update; \
+    apt-get install -y ./mount-s3.deb; \
+    rm -f ./mount-s3.deb
+
 
 # python packages
 RUN pip install boto3==1.15 awscli==1.18.152 botocore==1.18.11

awsf3-docker/run.sh

@@ -150,7 +150,7 @@ exl echo "## cwltool version $(cwltool --version | cut -f2 -d' ')"
 exl echo "## cromwell version $(java -jar /usr/local/bin/cromwell-35.jar --version | cut -f2 -d ' ') for WDL draft2"
 exl echo "## cromwell version $(java -jar /usr/local/bin/cromwell.jar --version | cut -f2 -d ' ') for WDL v1.0"
 exl echo "## $(singularity --version)"
-exl echo "## $(goofys --version 2>&1)"
+exl echo "## $(mount-s3 --version)"
 
 
 # getting run.json file
@@ -163,7 +163,7 @@ if [ -z "$S3_ENCRYPT_KEY_ID" ];
 then
   exl awsf3 decode_run_json -i $RUN_JSON_FILE_NAME
 else
-  exl awsf3 decode_run_json -i $RUN_JSON_FILE_NAME -k $S3_ENCRYPT_KEY_ID
+  exl awsf3 decode_run_json -i $RUN_JSON_FILE_NAME -k $S3_ENCRYPT_KEY_ID -r $INSTANCE_REGION
 fi
 
 

awsf3/__main__.py

@@ -33,7 +33,8 @@ def args(self):
         return {
             'decode_run_json':
                 [{'flag': ["-i", "--input-run-json"], 'help': "input run json file"},
-                 {'flag': ["-k", "--kms-key-id"], 'help': "kms-key-id to use for encrypting s3 files"}],
+                 {'flag': ["-k", "--kms-key-id"], 'help': "kms-key-id to use for encrypting s3 files"},
+                 {'flag': ["-r", "--region"], 'help': "AWS region"}],
             'download_workflow':
                 [],
             'update_postrun_json_init':
@@ -56,8 +57,8 @@ def args(self):
         }
 
 
-def decode_run_json(input_run_json, kms_key_id=None):
-    utils.decode_run_json(input_run_json, kms_key_id=kms_key_id)
+def decode_run_json(input_run_json, kms_key_id=None, region=None):
+    utils.decode_run_json(input_run_json, kms_key_id=kms_key_id, region=region)
 
 
 def download_workflow():

awsf3/utils.py

@@ -26,7 +26,7 @@
 INPUT_MOUNT_DIR_PREFIX = "/data1/input-mounted-"  # data are mounted to this directory + bucket name
 
 
-def decode_run_json(input_json_file, kms_key_id=None):
+def decode_run_json(input_json_file, kms_key_id=None, region=None):
     """reads a run json file and creates three text files:
     download command list file (commands to download input files from s3)
     input yml file (for cwl/wdl/snakemake run)
@@ -42,7 +42,7 @@ def decode_run_json(input_json_file, kms_key_id=None):
     create_download_command_list(downloadlist_filename, runjson_input)
 
     # create a bucket-mounting command list file
-    create_mount_command_list(mountlist_filename, runjson_input, kms_key_id=kms_key_id)
+    create_mount_command_list(mountlist_filename, runjson_input, kms_key_id=kms_key_id, region=region)
 
     # create an input yml file to be used on awsem
     if language in ['wdl', 'wdl_v1', 'wdl_draft2']:  # wdl
@@ -57,24 +57,31 @@ def decode_run_json(input_json_file, kms_key_id=None):
 
 
 def create_mount_command_list(mountlist_filename, runjson_input,
-                              kms_key_id=None):
-    """ This function creates a mount point directory and starts goofys
+                              kms_key_id=None, region=None):
+    """ This function creates a mount point directory and starts mountpoint-s3
         with some default arguments.
         Note that KMS key arguments are needed for the mount if encryption
         is enabled.
     """
+    key_arn = None
+    if kms_key_id and region:
+        kms = boto3.client("kms", region_name=region)
+        response = kms.describe_key(KeyId=kms_key_id)
+        key_arn = response["KeyMetadata"]["Arn"]
+        
     buckets_to_be_mounted = set()
     for category in ["Input_files_data", "Secondary_files_data"]:
         for inkey, v in getattr(runjson_input, category).items():
             if v.mount:
                 buckets_to_be_mounted.add(v.dir_)
     with open(mountlist_filename, 'w') as f:
-        for b in sorted(buckets_to_be_mounted):
-            f.write("mkdir -p %s\n" % (INPUT_MOUNT_DIR_PREFIX + b))
-            if kms_key_id:
-                f.write("goofys --sse-kms %s -f %s %s &\n" % (kms_key_id, b, INPUT_MOUNT_DIR_PREFIX + b))
+        for bucket in sorted(buckets_to_be_mounted):
+            path_to_mount = INPUT_MOUNT_DIR_PREFIX + bucket
+            f.write(f"mkdir -p {path_to_mount}\n")
+            if key_arn:
+                f.write(f"mount-s3 {bucket} {path_to_mount} --sse aws:kms --sse-kms-key-id {key_arn} &\n")
             else:
-                f.write("goofys -f %s %s &\n" % (b, INPUT_MOUNT_DIR_PREFIX + b))
+                f.write(f"mount-s3 {bucket} {path_to_mount} &\n")
 
 
 def create_download_command_list(downloadlist_filename, runjson_input):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tibanna"
-version = "5.5.3"
+version = "6.0.0"
 description = "Tibanna runs portable pipelines (in CWL/WDL) on the AWS Cloud."
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

tests/awsf3/postrunjson/GBPtlqb2rFGH.postrun.json

@@ -169,6 +169,7 @@
         "behavior_on_capacity_limit": "wait_and_retry",
         "cloudwatch_dashboard": false,
         "cpu": "",
+        "disable_metrics_collection": false,
         "ebs_iops": "",
         "ebs_size": 45,
         "ebs_size_as_is": false,

tests/awsf3/test_utils.py

@@ -297,9 +297,9 @@ def test_create_mount_command_list():
         mcfile_content = f.read()
 
     right_content = ('mkdir -p /data1/input-mounted-somebucket\n'
-                     'goofys -f somebucket /data1/input-mounted-somebucket &\n'
+                     'mount-s3 somebucket /data1/input-mounted-somebucket &\n'
                      'mkdir -p /data1/input-mounted-somebucket2\n'
-                     'goofys -f somebucket2 /data1/input-mounted-somebucket2 &\n')
+                     'mount-s3 somebucket2 /data1/input-mounted-somebucket2 &\n')
 
     assert mcfile_content == right_content
     os.remove(mountcommand_filename)

tibanna/top.py

@@ -66,8 +66,8 @@ class Top(object):
     # These commands are excluded when parsing the top output
     # Currently only 1-, 2- or 3-word prefixes work.
     exclude_list = ['top', 'docker', 'dockerd', '/usr/bin/dockerd', 'cron',
-                    'docker-untar', 'containerd', 'goofys-latest', 'cwltool',
-                    '/usr/bin/containerd-shim-runc-v2', 'goofys', 'nodejs --eval',
+                    'docker-untar', 'containerd', 'mount-s3', 'cwltool',
+                    '/usr/bin/containerd-shim-runc-v2', 'nodejs --eval',
                     '/usr/bin/python3 /usr/local/bin/cwltool', 'containerd-shim',
                     '/usr/bin/python3 /bin/unattended-upgrade',
                     '/usr/bin/python3 /usr/local/bin/awsf3',