From d3cc8eade22c2e209ed326b7510d154aaa3d2379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Thu, 2 Jan 2025 16:33:37 +0100 Subject: [PATCH 1/2] add s3_bucket_delete command use the same frame as the download command --- .../coal/cli/commands/s3_bucket_delete.py | 113 ++++++++++++++++++ cosmotech/coal/cli/main.py | 4 +- 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 cosmotech/coal/cli/commands/s3_bucket_delete.py diff --git a/cosmotech/coal/cli/commands/s3_bucket_delete.py b/cosmotech/coal/cli/commands/s3_bucket_delete.py new file mode 100644 index 00000000..4bf17459 --- /dev/null +++ b/cosmotech/coal/cli/commands/s3_bucket_delete.py @@ -0,0 +1,113 @@ +# Copyright (C) - 2023 - 2024 - Cosmo Tech +# This document and all information contained herein is the exclusive property - +# including all intellectual property rights pertaining thereto - of Cosmo Tech. +# Any use, reproduction, translation, broadcasting, transmission, distribution, +# etc., to any person is prohibited unless it has been previously and +# specifically authorized by written means by Cosmo Tech. + +from typing import Optional + +import boto3 + +from cosmotech.coal.cli.utils.click import click +from cosmotech.coal.cli.utils.decorators import web_help +from cosmotech.coal.utils.logger import LOGGER + + +@click.command() +@click.option("--bucket-name", + envvar="CSM_DATA_BUCKET_NAME", + help="The bucket on S3 to delete", + metavar="BUCKET", + type=str, + show_envvar=True, + required=True) +@click.option("--prefix-filter", + "file_prefix", + envvar="CSM_DATA_BUCKET_PREFIX", + help="A prefix by which all deleted files should start in the bucket", + metavar="PREFIX", + type=str, + show_envvar=True) +@click.option("--use-ssl/--no-ssl", + default=True, + help="Use SSL to secure connection to S3", + type=bool, + is_flag=True) +@click.option("--s3-url", + "endpoint_url", + help="URL to connect to the S3 system", + type=str, + required=True, + show_envvar=True, + metavar="URL", + envvar="AWS_ENDPOINT_URL") +@click.option("--access-id", + "access_id", + help="Identity used to connect to the S3 system", + type=str, + required=True, + show_envvar=True, + metavar="ID", + envvar="AWS_ACCESS_KEY_ID") +@click.option("--secret-key", + "secret_key", + help="Secret tied to the ID used to connect to the S3 system", + type=str, + required=True, + show_envvar=True, + metavar="ID", + envvar="AWS_SECRET_ACCESS_KEY") +@click.option("--ssl-cert-bundle", + help="Path to an alternate CA Bundle to validate SSL connections", + type=str, + show_envvar=True, + metavar="PATH", + envvar="CSM_S3_CA_BUNDLE") +@web_help("csm-data/s3-bucket-delete") +def s3_bucket_delete( + bucket_name: str, + file_prefix: str, + endpoint_url: str, + access_id: str, + secret_key: str, + use_ssl: bool = True, + ssl_cert_bundle: Optional[str] = None, +): + """Delete S3 bucket content to a given folder + +Will delete everything in the bucket unless a prefix is set, then only file following the given prefix will be deleted + +Make use of the boto3 library to access the bucket + +More information is available on this page: +[https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) +""" + boto3_parameters = { + "use_ssl": use_ssl, + "endpoint_url": endpoint_url, + "aws_access_key_id": access_id, + "aws_secret_access_key": secret_key, + } + if ssl_cert_bundle: + boto3_parameters["verify"] = ssl_cert_bundle + + s3_resource = boto3.resource("s3", + **boto3_parameters) + + bucket = s3_resource.Bucket(bucket_name) + + remove_prefix = False + if file_prefix: + bucket_files = bucket.objects.filter(Prefix=file_prefix) + if file_prefix.endswith("/"): + remove_prefix = True + else: + bucket_files = bucket.objects.all() + for _file in bucket_files: + if not (path_name := str(_file.key)).endswith("/"): + target_file = path_name + if remove_prefix: + target_file = target_file.removeprefix(file_prefix) + LOGGER.info(f"Deleting {path_name}") + bucket.delete_key(_file.key) diff --git a/cosmotech/coal/cli/main.py b/cosmotech/coal/cli/main.py index f90768bb..8ce82726 100644 --- a/cosmotech/coal/cli/main.py +++ b/cosmotech/coal/cli/main.py @@ -12,6 +12,7 @@ from cosmotech.coal.cli.commands.legacy.legacy import legacy from cosmotech.coal.cli.commands.s3_bucket_download import s3_bucket_download from cosmotech.coal.cli.commands.s3_bucket_upload import s3_bucket_upload +from cosmotech.coal.cli.commands.s3_bucket_delete import s3_bucket_delete from cosmotech.coal.cli.commands.store.store import store from cosmotech.coal.cli.utils.click import click from cosmotech.coal.cli.utils.decorators import web_help @@ -23,7 +24,7 @@ def print_version(ctx, param, value): return click.echo(f"Cosmo Tech Data Interface {__version__}") ctx.exit() - + @click.group("csm-data") @click_log.simple_verbosity_option(LOGGER, @@ -49,6 +50,7 @@ def main(): main.add_command(store, "store") main.add_command(s3_bucket_download, "s3-bucket-download") main.add_command(s3_bucket_upload, "s3-bucket-upload") +main.add_command(s3_bucket_delete, "s3-bucket-delete") main.add_command(adx_send_scenariodata, "adx-send-scenariodata") if __name__ == "__main__": From eafdc23171d8f6e940f3a207ca60cffc13c6ae78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Mon, 13 Jan 2025 17:48:06 +0100 Subject: [PATCH 2/2] add generated doc --- docs/csm-data/s3-bucket-delete.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/csm-data/s3-bucket-delete.md diff --git a/docs/csm-data/s3-bucket-delete.md b/docs/csm-data/s3-bucket-delete.md new file mode 100644 index 00000000..217a488c --- /dev/null +++ b/docs/csm-data/s3-bucket-delete.md @@ -0,0 +1,11 @@ +--- +hide: + - toc +description: "Command help: `csm-data s3-bucket-delete`" +--- +# s3-bucket-delete + +!!! info "Help command" + ```text + --8<-- "generated/commands_help/csm-data/s3-bucket-delete.txt" + ```