Integrate S3ObjectIndex functionality into gardenlinux.s3.Bucket

Signed-off-by: Tobias Wolf <[email protected]>

Author: NotTheEvilOne

src/gardenlinux/s3/__init__.py

@@ -6,6 +6,5 @@
 
 from .bucket import Bucket
 from .s3_artifacts import S3Artifacts
-from .s3_object_index import S3ObjectIndex
 
-__all__ = ["Bucket", "S3Artifacts", "S3ObjectIndex"]
+__all__ = ["Bucket", "S3Artifacts"]

src/gardenlinux/s3/bucket.py

@@ -4,10 +4,15 @@
 S3 bucket
 """
 
-import boto3
+import json
 import logging
+from os import PathLike
+from pathlib import Path
+from time import time
 from typing import Any, Optional
 
+import boto3
+
 from ..logger import LoggerSetup
 
 
@@ -111,6 +116,35 @@ def download_fileobj(self, key, fp, *args, **kwargs):
 
         self._logger.info(f"Downloaded {key} from S3 as binary data")
 
+    def read_cache_file_or_filter(self, cache_file, cache_ttl: int = 3600, **kwargs):
+        """
+        Read S3 object keys from cache if valid or filter for S3 object keys.
+
+        :param cache_file: Path to cache file
+        :param cache_ttl:  Cache time-to-live in seconds
+
+        :returns: S3 object keys read or filtered
+
+        :since: 0.9.0
+        """
+
+        if not isinstance(cache_file, PathLike):
+            cache_file = Path(cache_file)
+
+        if cache_file.exists() and (time() - cache_file.stat().st_mtime) < cache_ttl:
+            with cache_file.open("r") as fp:
+                return json.loads(fp.read())
+
+        artifacts = [
+            s3_object.key for s3_object in self._bucket.objects.filter(**kwargs).all()
+        ]
+
+        if cache_file is not None:
+            with cache_file.open("w") as fp:
+                fp.write(json.dumps(artifacts))
+
+        return artifacts
+
     def upload_file(self, file_name, key, *args, **kwargs):
         """
         boto3: Upload a file to an S3 object.

src/gardenlinux/s3/s3_artifacts.py

@@ -16,11 +16,10 @@
 from typing import Any, Optional
 from urllib.parse import urlencode
 
+from .bucket import Bucket
 from ..features.cname import CName
 from ..logger import LoggerSetup
 
-from .bucket import Bucket
-
 
 class S3Artifacts(object):
     """
@@ -55,6 +54,17 @@ def __init__(
 
         self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config)
 
+    @property
+    def bucket(self):
+        """
+        Returns the underlying S3 bucket.
+
+        :return: (boto3.Bucket) S3 bucket
+        :since:  0.9.0
+        """
+
+        return self._bucket
+
     def download_to_directory(
         self,
         cname,

src/gardenlinux/s3/s3_object_index.py

@@ -69,6 +69,29 @@ def test_download_file(s3_setup):
     assert target_path.read_text() == "some data"
 
 
+def test_read_cache_file_or_filter(s3_setup):
+    """
+    Try to read with cache
+    """
+
+    env = s3_setup
+    env.s3.Object(env.bucket_name, "file.txt").put(Body=b"some data")
+
+    bucket = Bucket(env.bucket_name, s3_resource_config={"region_name": REGION})
+    cache_file = env.tmp_path / "s3.cache.json"
+
+    result = bucket.read_cache_file_or_filter(cache_file, 1, Prefix="file")
+    assert result == ["file.txt"]
+
+    env.s3.Object(env.bucket_name, "file2.txt").put(Body=b"some data")
+
+    result = bucket.read_cache_file_or_filter(cache_file, 3600, Prefix="file")
+    assert result == ["file.txt"]
+
+    result = bucket.read_cache_file_or_filter(cache_file, 0, Prefix="file")
+    assert result == ["file.txt", "file2.txt"]
+
+
 def test_upload_fileobj(s3_setup):
     """
     Upload a file-like in-memory object to the bucket