Merge pull request #47 from Cosmo-Tech/LAL/s3_bucket_delete

add s3_bucket_delete command

Author: neomatamune

cosmotech/coal/cli/commands/s3_bucket_delete.py

@@ -0,0 +1,113 @@
+# Copyright (C) - 2023 - 2024 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from typing import Optional
+
+import boto3
+
+from cosmotech.coal.cli.utils.click import click
+from cosmotech.coal.cli.utils.decorators import web_help
+from cosmotech.coal.utils.logger import LOGGER
+
+
+@click.command()
+@click.option("--bucket-name",
+              envvar="CSM_DATA_BUCKET_NAME",
+              help="The bucket on S3 to delete",
+              metavar="BUCKET",
+              type=str,
+              show_envvar=True,
+              required=True)
+@click.option("--prefix-filter",
+              "file_prefix",
+              envvar="CSM_DATA_BUCKET_PREFIX",
+              help="A prefix by which all deleted files should start in the bucket",
+              metavar="PREFIX",
+              type=str,
+              show_envvar=True)
+@click.option("--use-ssl/--no-ssl",
+              default=True,
+              help="Use SSL to secure connection to S3",
+              type=bool,
+              is_flag=True)
+@click.option("--s3-url",
+              "endpoint_url",
+              help="URL to connect to the S3 system",
+              type=str,
+              required=True,
+              show_envvar=True,
+              metavar="URL",
+              envvar="AWS_ENDPOINT_URL")
+@click.option("--access-id",
+              "access_id",
+              help="Identity used to connect to the S3 system",
+              type=str,
+              required=True,
+              show_envvar=True,
+              metavar="ID",
+              envvar="AWS_ACCESS_KEY_ID")
+@click.option("--secret-key",
+              "secret_key",
+              help="Secret tied to the ID used to connect to the S3 system",
+              type=str,
+              required=True,
+              show_envvar=True,
+              metavar="ID",
+              envvar="AWS_SECRET_ACCESS_KEY")
+@click.option("--ssl-cert-bundle",
+              help="Path to an alternate CA Bundle to validate SSL connections",
+              type=str,
+              show_envvar=True,
+              metavar="PATH",
+              envvar="CSM_S3_CA_BUNDLE")
+@web_help("csm-data/s3-bucket-delete")
+def s3_bucket_delete(
+    bucket_name: str,
+    file_prefix: str,
+    endpoint_url: str,
+    access_id: str,
+    secret_key: str,
+    use_ssl: bool = True,
+    ssl_cert_bundle: Optional[str] = None,
+):
+    """Delete S3 bucket content to a given folder
+
+Will delete everything in the bucket unless a prefix is set, then only file following the given prefix will be deleted
+
+Make use of the boto3 library to access the bucket
+
+More information is available on this page:
+[https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html)
+"""
+    boto3_parameters = {
+        "use_ssl": use_ssl,
+        "endpoint_url": endpoint_url,
+        "aws_access_key_id": access_id,
+        "aws_secret_access_key": secret_key,
+    }
+    if ssl_cert_bundle:
+        boto3_parameters["verify"] = ssl_cert_bundle
+
+    s3_resource = boto3.resource("s3",
+                                 **boto3_parameters)
+
+    bucket = s3_resource.Bucket(bucket_name)
+
+    remove_prefix = False
+    if file_prefix:
+        bucket_files = bucket.objects.filter(Prefix=file_prefix)
+        if file_prefix.endswith("/"):
+            remove_prefix = True
+    else:
+        bucket_files = bucket.objects.all()
+    for _file in bucket_files:
+        if not (path_name := str(_file.key)).endswith("/"):
+            target_file = path_name
+            if remove_prefix:
+                target_file = target_file.removeprefix(file_prefix)
+            LOGGER.info(f"Deleting {path_name}")
+            bucket.delete_key(_file.key)

cosmotech/coal/cli/main.py

@@ -12,6 +12,7 @@
 from cosmotech.coal.cli.commands.legacy.legacy import legacy
 from cosmotech.coal.cli.commands.s3_bucket_download import s3_bucket_download
 from cosmotech.coal.cli.commands.s3_bucket_upload import s3_bucket_upload
+from cosmotech.coal.cli.commands.s3_bucket_delete import s3_bucket_delete
 from cosmotech.coal.cli.commands.store.store import store
 from cosmotech.coal.cli.utils.click import click
 from cosmotech.coal.cli.utils.decorators import web_help
@@ -23,7 +24,7 @@ def print_version(ctx, param, value):
         return
     click.echo(f"Cosmo Tech Data Interface {__version__}")
     ctx.exit()
-    
+
 
 @click.group("csm-data")
 @click_log.simple_verbosity_option(LOGGER,
@@ -49,6 +50,7 @@ def main():
 main.add_command(store, "store")
 main.add_command(s3_bucket_download, "s3-bucket-download")
 main.add_command(s3_bucket_upload, "s3-bucket-upload")
+main.add_command(s3_bucket_delete, "s3-bucket-delete")
 main.add_command(adx_send_scenariodata, "adx-send-scenariodata")
 
 if __name__ == "__main__":

docs/csm-data/s3-bucket-delete.md

@@ -0,0 +1,11 @@
+---
+hide:
+  - toc
+description: "Command help: `csm-data s3-bucket-delete`"
+---
+# s3-bucket-delete
+
+!!! info "Help command"
+    ```text
+    --8<-- "generated/commands_help/csm-data/s3-bucket-delete.txt"
+    ```