Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Pipfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true

[dev-packages]

[packages]
black = "*"
ipython = "*"
boto3 = "*"

[requires]
python_version = "3.7"

[pipenv]
allow_prereleases = true
219 changes: 219 additions & 0 deletions Pipfile.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file added pizzas.zip
Binary file not shown.
4 changes: 4 additions & 0 deletions roman-candle.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
name,quantity
Roman Candle,2
Veggie Candle,1
Marco Pollo Parm,1
43 changes: 43 additions & 0 deletions warmup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import io
import os
import zipfile
from pathlib import Path
import boto3


def unzip_object_in_s3(bucket, zip_key_name, file_key_name):
"""Download, unzip, and upload a file from a zip archive in an S3 bucket.

Performs the unzipping in memory.

Args:
bucket: A boto3 Resource object for an S3 bucket
zip_key_name: The key to the zip file in the bucket
file_key_name: The name of the file in the zip archive to extract
Returns:
dst_key_name: The key to the extracted file in the bucket
"""
dst_key_name = Path(zip_key_name).parent / Path(zip_key_name).stem / file_key_name
with io.BytesIO() as compressed:
s3_bucket.download_fileobj(Key=zip_key_name, Fileobj=compressed)
with zipfile.ZipFile(compressed) as zip_handle:
with zip_handle.open(file_key_name) as decompressed:
s3_bucket.upload_fileobj(
Fileobj=decompressed, Key=str(dst_key_name)
)

return str(dst_key_name)


assert os.environ["AWS_PROFILE"]
# aws s3 cp pizzas.zip s3://madpy/warmup/2019-10-10/pizzas.zip
s3_bucket = boto3.resource("s3").Bucket("madpy")
zip_key_name = "warmup/2019-10-10/pizzas.zip"
file_key_name = "roman-candle.csv"
dst_key_name = unzip_object_in_s3(s3_bucket, zip_key_name, file_key_name)


with io.BytesIO() as test:
s3_bucket.download_fileobj(Key=dst_key_name, Fileobj=test)
test.seek(0)
print(test.read().decode("utf-8"))