From a084b873b21456785691623b9a16ba7ea2f1a2d8 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd Date: Tue, 8 Apr 2025 17:07:08 -0700 Subject: [PATCH 1/6] Sample script to compare --- scripts/encryption_validator/readme.md | 23 ++++++++ scripts/encryption_validator/script.py | 78 ++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 scripts/encryption_validator/readme.md create mode 100644 scripts/encryption_validator/script.py diff --git a/scripts/encryption_validator/readme.md b/scripts/encryption_validator/readme.md new file mode 100644 index 00000000..b052756c --- /dev/null +++ b/scripts/encryption_validator/readme.md @@ -0,0 +1,23 @@ + +# Salt Compare Tool + +A lightweight Python script to compare encrypted and unencrypted salt files stored in an S3 bucket. + +## Description + +This script fetches two S3 objects (e.g. encrypted and unencrypted salts) and compares them. It validates basic input rules like prefix formatting and key pattern before proceeding. + +## Usage + +### Run the script: + +`python script.py [prefix]` + +- `key` – Required. Must start with `salt`. Example: `salts/salts.txt.1744152738842` +- `bucket` – Required. Name of the S3 bucket. +- `region_name` – Required. AWS region of the S3 bucket (e.g. `us-east-1`) +- `prefix` – Optional. S3 path prefix. If provided, it **must end with `/`**. + +## For Other Decryption Comparisons + +You can use the **same logic** for other types of decryption and comparison. The only change is in how the **unencrypted file name** is generated in salt_compare. \ No newline at end of file diff --git a/scripts/encryption_validator/script.py b/scripts/encryption_validator/script.py new file mode 100644 index 00000000..cd2ffbc7 --- /dev/null +++ b/scripts/encryption_validator/script.py @@ -0,0 +1,78 @@ +import json +import base64 +from typing import IO +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend +import boto3 +import sys + +class AesGcm: + @staticmethod + def decrypt(encrypted_data: bytes, nonce: bytes, key: bytes): + if len(nonce) != 12: + raise ValueError("Nonce must be 12 bytes for AES-GCM") + cipher = Cipher(algorithms.AES(key), modes.GCM(nonce), backend=default_backend()) + decryptor = cipher.decryptor() + try: + return decryptor.update(encrypted_data) + decryptor.finalize() + except Exception: + raise ValueError("Invalid GCM tag during decryption") + +def _get_encryption_secret(key_id, bucket, prefix, region_name): + s3 = boto3.client('s3', region_name=region_name) + response = s3.get_object(Bucket=bucket, Key=f"{prefix}cloud_encryption_keys/cloud_encryption_keys.json") + data = json.load(response['Body']) + _map = {item['id']: item for item in data} + return _map.get(key_id).get('secret') + +def _decrypt_input_stream(input_stream: IO[bytes], bucket, prefix, region_name) -> str: + try: + data = json.load(input_stream) + except json.JSONDecodeError as e: + raise ValueError(f"Failed to parse JSON: {e}") + key_id = data.get("key_id") + encrypted_payload_b64 = data.get("encrypted_payload") + if key_id is None or encrypted_payload_b64 is None: + raise ValueError("Failed to parse JSON") + + decryption_key = _get_encryption_secret(key_id, bucket, prefix, region_name) + try: + secret_bytes = base64.b64decode(decryption_key) + encrypted_bytes = base64.b64decode(encrypted_payload_b64) + nonce = encrypted_bytes[:12] + ciphertext = encrypted_bytes[12:] + ciphertext = encrypted_bytes[12:-16] + auth_tag = encrypted_bytes[-16:] + cipher = Cipher(algorithms.AES(secret_bytes), modes.GCM(nonce, auth_tag), backend=default_backend()) + decryptor = cipher.decryptor() + decrypted_bytes = decryptor.update(ciphertext) + decryptor.finalize() + return decrypted_bytes.decode("utf-8") + except Exception as e: + raise ValueError(f"An error occurred during decryption: {e}") + +def salt_compare(key, prefix, bucket, region_name): + s3 = boto3.client('s3', region_name=region_name) + key = f"{prefix}{key}" + base_path = '/'.join(key.split('/')[:-3]) + file_name = key.split('/')[-1:][0] + unencrypted = f'{base_path}/{file_name}' + print(f"Comparing {key} with {unencrypted}") + response = s3.get_object(Bucket=bucket, Key=key) + encrypted = _decrypt_input_stream(response['Body'], bucket=bucket, prefix=prefix, region_name=region_name) + response = s3.get_object(Bucket=bucket, Key=unencrypted) + unencrypted = response['Body'].read().decode('utf-8') + return (encrypted==unencrypted) + +if __name__ == '__main__': + key = sys.argv[1] + bucket = sys.argv[2] + region_name = sys.argv[3] + prefix = sys.argv[4] if len(sys.argv) > 4 else '' + if prefix != '' and prefix[-1]!='/': + raise "prefix should terminate with /" + if not key.startswith("salt"): + raise "only salts supported" + print(salt_compare(key=key, prefix=prefix, bucket=bucket, region_name=region_name)) + + + \ No newline at end of file From 32fb177b183f129d9ca571a3039a568217285a51 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd Date: Tue, 8 Apr 2025 17:07:46 -0700 Subject: [PATCH 2/6] Sample script to compare --- scripts/encryption_validator/script.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/encryption_validator/script.py b/scripts/encryption_validator/script.py index cd2ffbc7..5e4b02ab 100644 --- a/scripts/encryption_validator/script.py +++ b/scripts/encryption_validator/script.py @@ -73,6 +73,3 @@ def salt_compare(key, prefix, bucket, region_name): if not key.startswith("salt"): raise "only salts supported" print(salt_compare(key=key, prefix=prefix, bucket=bucket, region_name=region_name)) - - - \ No newline at end of file From cc4d150d7e5e972e7fbcac4e6e29ad11086cf197 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd Date: Tue, 8 Apr 2025 17:09:39 -0700 Subject: [PATCH 3/6] Sample script to compare --- scripts/encryption_validator/readme.md | 2 ++ scripts/encryption_validator/requirements.txt | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 scripts/encryption_validator/requirements.txt diff --git a/scripts/encryption_validator/readme.md b/scripts/encryption_validator/readme.md index b052756c..0e6af9c7 100644 --- a/scripts/encryption_validator/readme.md +++ b/scripts/encryption_validator/readme.md @@ -11,6 +11,8 @@ This script fetches two S3 objects (e.g. encrypted and unencrypted salts) and co ### Run the script: +Login to AWS account +`pip install requirements.txt` `python script.py [prefix]` - `key` – Required. Must start with `salt`. Example: `salts/salts.txt.1744152738842` diff --git a/scripts/encryption_validator/requirements.txt b/scripts/encryption_validator/requirements.txt new file mode 100644 index 00000000..a7adccb5 --- /dev/null +++ b/scripts/encryption_validator/requirements.txt @@ -0,0 +1,2 @@ +cryptography==39.0.1 +boto3==1.26.11 \ No newline at end of file From 6cf089943c602bf8e337f6dd78ec98b14e48899e Mon Sep 17 00:00:00 2001 From: abuabraham-ttd Date: Tue, 8 Apr 2025 17:10:46 -0700 Subject: [PATCH 4/6] Sample script to compare --- scripts/encryption_validator/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/encryption_validator/script.py b/scripts/encryption_validator/script.py index 5e4b02ab..5412195a 100644 --- a/scripts/encryption_validator/script.py +++ b/scripts/encryption_validator/script.py @@ -19,6 +19,7 @@ def decrypt(encrypted_data: bytes, nonce: bytes, key: bytes): raise ValueError("Invalid GCM tag during decryption") def _get_encryption_secret(key_id, bucket, prefix, region_name): + print("Fetching secret key for ", key_id) s3 = boto3.client('s3', region_name=region_name) response = s3.get_object(Bucket=bucket, Key=f"{prefix}cloud_encryption_keys/cloud_encryption_keys.json") data = json.load(response['Body']) @@ -40,7 +41,6 @@ def _decrypt_input_stream(input_stream: IO[bytes], bucket, prefix, region_name) secret_bytes = base64.b64decode(decryption_key) encrypted_bytes = base64.b64decode(encrypted_payload_b64) nonce = encrypted_bytes[:12] - ciphertext = encrypted_bytes[12:] ciphertext = encrypted_bytes[12:-16] auth_tag = encrypted_bytes[-16:] cipher = Cipher(algorithms.AES(secret_bytes), modes.GCM(nonce, auth_tag), backend=default_backend()) From d08a9c60de25bbadc137a9ce96deef825ccdda9f Mon Sep 17 00:00:00 2001 From: abuabraham-ttd Date: Tue, 8 Apr 2025 17:16:26 -0700 Subject: [PATCH 5/6] dont pin version --- scripts/encryption_validator/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/encryption_validator/requirements.txt b/scripts/encryption_validator/requirements.txt index a7adccb5..3767bee6 100644 --- a/scripts/encryption_validator/requirements.txt +++ b/scripts/encryption_validator/requirements.txt @@ -1,2 +1,2 @@ -cryptography==39.0.1 -boto3==1.26.11 \ No newline at end of file +cryptography +boto3 \ No newline at end of file From 5548467f31e1134049e6877dd6fd3cbc9110c548 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd Date: Wed, 23 Apr 2025 15:28:41 -0700 Subject: [PATCH 6/6] Query n files, change key to encrypted_file --- scripts/encryption_validator/readme.md | 2 +- scripts/encryption_validator/script.py | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/encryption_validator/readme.md b/scripts/encryption_validator/readme.md index 0e6af9c7..ea2539ae 100644 --- a/scripts/encryption_validator/readme.md +++ b/scripts/encryption_validator/readme.md @@ -15,7 +15,7 @@ Login to AWS account `pip install requirements.txt` `python script.py [prefix]` -- `key` – Required. Must start with `salt`. Example: `salts/salts.txt.1744152738842` +- `encrypted_file` – Required. Must start with `salt`. Example: `salts/encrypted/12_private/salts.txt.1745532777048` (To query multiple files you can use `salts/encrypted/12_private/*`) - `bucket` – Required. Name of the S3 bucket. - `region_name` – Required. AWS region of the S3 bucket (e.g. `us-east-1`) - `prefix` – Optional. S3 path prefix. If provided, it **must end with `/`**. diff --git a/scripts/encryption_validator/script.py b/scripts/encryption_validator/script.py index 5412195a..eaef72ce 100644 --- a/scripts/encryption_validator/script.py +++ b/scripts/encryption_validator/script.py @@ -53,6 +53,7 @@ def _decrypt_input_stream(input_stream: IO[bytes], bucket, prefix, region_name) def salt_compare(key, prefix, bucket, region_name): s3 = boto3.client('s3', region_name=region_name) key = f"{prefix}{key}" + print("Key is ", key) base_path = '/'.join(key.split('/')[:-3]) file_name = key.split('/')[-1:][0] unencrypted = f'{base_path}/{file_name}' @@ -63,13 +64,31 @@ def salt_compare(key, prefix, bucket, region_name): unencrypted = response['Body'].read().decode('utf-8') return (encrypted==unencrypted) +def _get_most_recent_files(bucket, prefix, key): + s3 = boto3.client("s3") + paginator = s3.get_paginator("list_objects_v2") + page_iterator = paginator.paginate(Bucket=bucket, Prefix=f"{prefix}{key[:-2]}/") + n, all_files = 5 , [] + for i, page in enumerate(page_iterator): + if i >= n: + break + all_files.extend(page.get("Contents", [])) + recent_files = sorted(all_files, key=lambda x: x["LastModified"], reverse=True) + recent_files = list(map(lambda x: x['Key'], recent_files)) + recent_files = list(filter(lambda x: "metadata" not in x, recent_files)) + return recent_files[:10] + if __name__ == '__main__': - key = sys.argv[1] + encrypted_file = sys.argv[1] bucket = sys.argv[2] region_name = sys.argv[3] prefix = sys.argv[4] if len(sys.argv) > 4 else '' if prefix != '' and prefix[-1]!='/': raise "prefix should terminate with /" - if not key.startswith("salt"): + if not encrypted_file.startswith("salt"): raise "only salts supported" - print(salt_compare(key=key, prefix=prefix, bucket=bucket, region_name=region_name)) + if encrypted_file[-2:] == '/*': + for recent in _get_most_recent_files(bucket=bucket, prefix=prefix, key=encrypted_file): + print(salt_compare(key=recent, prefix=prefix, bucket=bucket, region_name=region_name)) + else: + print(salt_compare(key=encrypted_file, prefix=prefix, bucket=bucket, region_name=region_name))