Skip to content

Commit b89339b

Browse files
vivus-ignisvivus-ignis
committed
proper mockups for s3-related tests + retrying setup
1 parent 199e2ca commit b89339b

File tree

5 files changed

+89
-25
lines changed

5 files changed

+89
-25
lines changed

‎src/gardenlinux/constants.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,3 +161,8 @@
161161
# https://github.com/gardenlinux/gardenlinux/issues/3044
162162
# Empty string is the 'legacy' variant with traditional root fs and still needed/supported
163163
IMAGE_VARIANTS = ["", "_usi", "_tpm2_trustedboot"]
164+
165+
# configuration for https://github.com/groodt/retrying
166+
RETRYING_MAX_ATTEMPTS = 5
167+
RETRYING_WAIT_EXPONENTIAL_MULTIPLIER = 1000
168+
RETRYING_WAIT_EXPONENTIAL_MAX = 16000

‎src/gardenlinux/s3/bucket.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import boto3
1515
from retrying import retry
1616

17+
from ..constants import RETRYING_MAX_ATTEMPTS, RETRYING_WAIT_EXPONENTIAL_MAX, RETRYING_WAIT_EXPONENTIAL_MULTIPLIER
1718
from ..logger import LoggerSetup
1819

1920

@@ -89,7 +90,9 @@ class tree for self).
8990

9091
return getattr(self._bucket, name)
9192

92-
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000, wait_exponential_max=16000)
93+
@retry(stop_max_attempt_number=RETRYING_MAX_ATTEMPTS,
94+
wait_exponential_multiplier=RETRYING_WAIT_EXPONENTIAL_MULTIPLIER,
95+
wait_exponential_max=RETRYING_WAIT_EXPONENTIAL_MAX)
9396
def download_file(self, key, file_name, *args, **kwargs):
9497
"""
9598
boto3: Download an S3 object to a file.

‎tests/constants.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
TEST_GARDENLINUX_RELEASE = "1877.3"
2626
TEST_GARDENLINUX_COMMIT = "75df9f401a842914563f312899ec3ce34b24515c"
2727
TEST_GARDENLINUX_COMMIT_SHORT = TEST_GARDENLINUX_COMMIT[:8]
28+
TEST_GARDENLINUX_RELEASE_BUCKET_NAME = "test__gardenlinux__releases"
2829

2930
RELEASE_NOTES_TEST_DATA_DIR = Path(os.path.dirname(__file__)) / ".." / "test-data" / "release_notes"
3031
RELEASE_NOTES_S3_ARTIFACTS_DIR = RELEASE_NOTES_TEST_DATA_DIR / "s3_bucket_artifacts"

‎tests/github/conftest.py‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
import pytest
77
from moto import mock_aws
88

9-
from gardenlinux.constants import GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME, RELEASE_ID_FILE, S3_DOWNLOADS_DIR
9+
from gardenlinux.constants import RELEASE_ID_FILE, S3_DOWNLOADS_DIR
10+
11+
from ..constants import TEST_GARDENLINUX_RELEASE_BUCKET_NAME
1012

1113

1214
@pytest.fixture
@@ -42,10 +44,39 @@ def release_id_file():
4244
def release_s3_bucket():
4345
with mock_aws():
4446
s3 = boto3.resource("s3", region_name="eu-central-1")
45-
s3.create_bucket(Bucket=GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME,
46-
CreateBucketConfiguration={
47-
'LocationConstraint': 'eu-central-1'})
48-
yield s3.Bucket(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
47+
s3.create_bucket(Bucket=TEST_GARDENLINUX_RELEASE_BUCKET_NAME,
48+
CreateBucketConfiguration={"LocationConstraint": "eu-central-1"})
49+
yield s3.Bucket(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
50+
51+
52+
@pytest.fixture
53+
def blackhole_s3_bucket():
54+
"""This fixture yields an object that behaves as an S3 bucket where
55+
any object can be found, but downloading a file always raises an exception.
56+
This is needed to test the retry mechanism as the object also counts
57+
how many times an exception was raised."""
58+
class BlackHoleObject():
59+
def __init__(self, bucket_name, key):
60+
self.bucket_name = bucket_name
61+
self.key = key
62+
63+
class BlackHoleObjects():
64+
def __init__(self, bucket_name):
65+
self.bucket_name = bucket_name
66+
67+
def filter(self, Prefix):
68+
return [BlackHoleObject(self.bucket_name, Prefix)]
69+
70+
class BlackHoleS3Bucket():
71+
def __init__(self, bucket_name):
72+
self.objects = BlackHoleObjects(bucket_name)
73+
self.download_attempts = 0
74+
75+
def download_file(self, x, y):
76+
self.download_attempts += 1
77+
raise IOError(f"Download attempt # {self.download_attempts} failed")
78+
79+
yield BlackHoleS3Bucket(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
4980

5081

5182
@pytest.fixture

‎tests/github/test_download_metadata_files.py‎

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
import pytest
22

3-
from gardenlinux.constants import (
4-
GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME,
5-
S3_DOWNLOADS_DIR,
6-
)
3+
from gardenlinux.constants import RETRYING_MAX_ATTEMPTS, S3_DOWNLOADS_DIR
74
from gardenlinux.features import CName
85
from gardenlinux.github.release_notes.helpers import download_metadata_file
96
from gardenlinux.s3 import S3Artifacts
@@ -13,68 +10,95 @@
1310
TEST_GARDENLINUX_COMMIT,
1411
TEST_GARDENLINUX_COMMIT_SHORT,
1512
TEST_GARDENLINUX_RELEASE,
13+
TEST_GARDENLINUX_RELEASE_BUCKET_NAME,
1614
)
1715

1816

1917
def test_download_metadata_file(downloads_dir, release_s3_bucket):
2018
release_s3_bucket.upload_file(RELEASE_NOTES_S3_ARTIFACTS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml",
21-
f"meta/singles/aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
19+
f"meta/singles/test-aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
20+
21+
s3_artifacts = S3Artifacts(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
22+
s3_artifacts._bucket = release_s3_bucket
2223

23-
s3_artifacts = S3Artifacts(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
24-
cname = CName("aws-gardener_prod", "amd64", "{0}-{1}".format(TEST_GARDENLINUX_RELEASE, TEST_GARDENLINUX_COMMIT_SHORT))
24+
cname = CName("test-aws-gardener_prod", "amd64", "{0}-{1}".format(TEST_GARDENLINUX_RELEASE, TEST_GARDENLINUX_COMMIT_SHORT))
2525
download_metadata_file(s3_artifacts,
2626
cname.cname,
2727
TEST_GARDENLINUX_RELEASE,
2828
TEST_GARDENLINUX_COMMIT_SHORT,
2929
S3_DOWNLOADS_DIR)
30-
assert (S3_DOWNLOADS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml").exists()
30+
assert (S3_DOWNLOADS_DIR / "test-aws-gardener_prod-amd64.s3_metadata.yaml").exists()
3131

3232

3333
def test_download_metadata_file_no_such_release(downloads_dir, release_s3_bucket):
3434
release_s3_bucket.upload_file(RELEASE_NOTES_S3_ARTIFACTS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml",
35-
f"meta/singles/aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
36-
s3_artifacts = S3Artifacts(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
35+
f"meta/singles/test-aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
36+
s3_artifacts = S3Artifacts(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
37+
s3_artifacts._bucket = release_s3_bucket
38+
3739
release = "0000.0"
3840
commit = TEST_GARDENLINUX_COMMIT_SHORT
3941
cname = CName("aws-gardener_prod", "amd64", "{0}-{1}".format(release, commit))
42+
4043
with pytest.raises(IndexError):
4144
download_metadata_file(s3_artifacts,
4245
cname.cname,
4346
release,
4447
commit,
4548
S3_DOWNLOADS_DIR)
46-
assert not (S3_DOWNLOADS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml").exists()
49+
assert not (S3_DOWNLOADS_DIR / "test-aws-gardener_prod-amd64.s3_metadata.yaml").exists()
4750

4851

4952
def test_download_metadata_file_no_such_commit(downloads_dir, release_s3_bucket):
5053
release_s3_bucket.upload_file(RELEASE_NOTES_S3_ARTIFACTS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml",
51-
f"meta/singles/aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
54+
f"meta/singles/test-aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
55+
56+
s3_artifacts = S3Artifacts(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
57+
s3_artifacts._bucket = release_s3_bucket
5258

53-
s3_artifacts = S3Artifacts(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
5459
release = TEST_GARDENLINUX_RELEASE
5560
commit = "deadbeef"
56-
cname = CName("aws-gardener_prod", "amd64", "{0}-{1}".format(release, commit))
61+
cname = CName("test-aws-gardener_prod", "amd64", "{0}-{1}".format(release, commit))
62+
5763
with pytest.raises(IndexError):
5864
download_metadata_file(s3_artifacts,
5965
cname.cname,
6066
release,
6167
commit,
6268
S3_DOWNLOADS_DIR)
63-
assert not (S3_DOWNLOADS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml").exists()
69+
assert not (S3_DOWNLOADS_DIR / "test-aws-gardener_prod-amd64.s3_metadata.yaml").exists()
6470

6571

6672
def test_download_metadata_file_no_such_release_and_commit(downloads_dir, release_s3_bucket):
6773
release_s3_bucket.upload_file(RELEASE_NOTES_S3_ARTIFACTS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml",
68-
f"meta/singles/aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
74+
f"meta/singles/test-aws-gardener_prod-amd64-{TEST_GARDENLINUX_RELEASE}-{TEST_GARDENLINUX_COMMIT}")
75+
76+
s3_artifacts = S3Artifacts(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
77+
s3_artifacts._bucket = release_s3_bucket
6978

70-
s3_artifacts = S3Artifacts(GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME)
7179
release = "0000.0"
7280
commit = "deadbeef"
73-
cname = CName("aws-gardener_prod", "amd64", "{0}-{1}".format(release, commit))
81+
cname = CName("test-aws-gardener_prod", "amd64", "{0}-{1}".format(release, commit))
82+
7483
with pytest.raises(IndexError):
7584
download_metadata_file(s3_artifacts,
7685
cname.cname,
7786
release,
7887
commit,
7988
S3_DOWNLOADS_DIR)
80-
assert not (S3_DOWNLOADS_DIR / "aws-gardener_prod-amd64.s3_metadata.yaml").exists()
89+
assert not (S3_DOWNLOADS_DIR / "test-aws-gardener_prod-amd64.s3_metadata.yaml").exists()
90+
91+
92+
def test_download_metadata_uses_retrying_strategy(downloads_dir, blackhole_s3_bucket):
93+
s3_artifacts = S3Artifacts(TEST_GARDENLINUX_RELEASE_BUCKET_NAME)
94+
s3_artifacts._bucket._bucket = blackhole_s3_bucket
95+
96+
cname = CName("test-aws-gardener_prod", "amd64", "{0}-{1}".format("foo", "bar"))
97+
98+
with pytest.raises(IOError) as exn:
99+
download_metadata_file(s3_artifacts,
100+
cname.cname,
101+
"foo",
102+
"bar",
103+
S3_DOWNLOADS_DIR)
104+
assert str(exn.value) == f"Download attempt # {RETRYING_MAX_ATTEMPTS} failed"

0 commit comments

Comments
 (0)