Skip to content

Commit 367e831

Browse files
ligangtyligangty
committed
Feat: multi-targets support: maven parameters change
1 parent 9f72670 commit 367e831

File tree

8 files changed

+57
-62
lines changed

8 files changed

+57
-62
lines changed

‎charon/cmd/command.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,10 @@ def upload(
175175
product_key,
176176
ignore_patterns_list,
177177
root=root_path,
178-
targets=[(aws_bucket, prefix_)],
178+
targets=[(target, aws_bucket, prefix_)],
179179
aws_profile=aws_profile,
180180
dir_=work_dir,
181181
dry_run=dryrun,
182-
manifest_folder=target,
183182
manifest_bucket_name=manifest_bucket_name
184183
)
185184
if not succeeded:
@@ -335,11 +334,10 @@ def delete(
335334
product_key,
336335
ignore_patterns_list,
337336
root=root_path,
338-
targets=[(aws_bucket, prefix_)],
337+
targets=[(target, aws_bucket, prefix_)],
339338
aws_profile=aws_profile,
340339
dir_=work_dir,
341340
dry_run=dryrun,
342-
manifest_folder=target,
343341
manifest_bucket_name=manifest_bucket_name
344342
)
345343
if not succeeded:

‎charon/pkgs/maven.py‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -256,12 +256,11 @@ def handle_maven_uploading(
256256
prod_key: str,
257257
ignore_patterns=None,
258258
root="maven-repository",
259-
targets: List[Tuple[str, str]] = None,
259+
targets: List[Tuple[str, str, str]] = None,
260260
aws_profile=None,
261261
dir_=None,
262262
do_index=True,
263263
dry_run=False,
264-
manifest_folder=None,
265264
manifest_bucket_name=None
266265
) -> Tuple[str, bool]:
267266
""" Handle the maven product release tarball uploading process.
@@ -273,9 +272,9 @@ def handle_maven_uploading(
273272
need to upload in the tarball
274273
* root is a prefix in the tarball to identify which path is
275274
the beginning of the maven GAV path
276-
* target is the a tuple of s3 bucket name and prefix for the bucket,
277-
whic will be used to store artifacts with the prefix. See target
278-
definition in Charon configuration for details
275+
* targets contains the target name with its bucket name and prefix
276+
for the bucket, which will be used to store artifacts with the
277+
prefix. See target definition in Charon configuration for details
279278
* dir_ is base dir for extracting the tarball, will use system
280279
tmp dir if None.
281280
@@ -305,39 +304,40 @@ def handle_maven_uploading(
305304
# Question: should we exit here?
306305

307306
main_target = targets[0]
308-
main_target_prefix = remove_prefix(main_target[1], "/")
307+
main_bucket_prefix = remove_prefix(main_target[2], "/")
309308
succeeded = True
310309
# 4. Do uploading
311310
logger.info("Start uploading files to s3")
312311
s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
313-
bucket = main_target[0]
312+
bucket = main_target[1]
314313
failed_files = s3_client.upload_files(
315314
file_paths=valid_mvn_paths,
316-
target=(bucket, main_target_prefix),
315+
target=(bucket, main_bucket_prefix),
317316
product=prod_key,
318317
root=top_level
319318
)
320319
logger.info("Files uploading done\n")
321320

322-
# 5. Do manifest uploading
323-
logger.info("Start uploading manifest to s3")
324-
if not manifest_bucket_name:
325-
logger.warning(
326-
'Warning: No manifest bucket is provided, will ignore the process of manifest '
327-
'uploading\n')
328-
else:
329-
manifest_name, manifest_full_path = write_manifest(valid_mvn_paths, top_level, prod_key)
330-
s3_client.upload_manifest(
331-
manifest_name, manifest_full_path,
332-
manifest_folder, manifest_bucket_name
333-
)
334-
logger.info("Manifest uploading is done\n")
335-
336-
# 6. Use uploaded poms to scan s3 for metadata refreshment
337321
all_failed_metas: Dict[str, List[str]] = {}
338322
for target in targets:
339-
bucket_ = target[0]
340-
prefix__ = remove_prefix(target[1], "/")
323+
# 5. Do manifest uploading
324+
logger.info("Start uploading manifest to s3")
325+
manifest_folder = target[0]
326+
if not manifest_bucket_name:
327+
logger.warning(
328+
'Warning: No manifest bucket is provided, will ignore the process of manifest '
329+
'uploading\n')
330+
else:
331+
manifest_name, manifest_full_path = write_manifest(valid_mvn_paths, top_level, prod_key)
332+
s3_client.upload_manifest(
333+
manifest_name, manifest_full_path,
334+
manifest_folder, manifest_bucket_name
335+
)
336+
logger.info("Manifest uploading is done\n")
337+
338+
# 6. Use uploaded poms to scan s3 for metadata refreshment
339+
bucket_ = target[1]
340+
prefix__ = remove_prefix(target[2], "/")
341341
failed_metas = []
342342
logger.info("Start generating maven-metadata.xml files for bucket %s", bucket_)
343343
meta_files = _generate_metadatas(
@@ -421,12 +421,11 @@ def handle_maven_del(
421421
prod_key: str,
422422
ignore_patterns=None,
423423
root="maven-repository",
424-
targets: Tuple[str, str] = None,
424+
targets: List[Tuple[str, str, str]] = None,
425425
aws_profile=None,
426426
dir_=None,
427427
do_index=True,
428428
dry_run=False,
429-
manifest_folder=None,
430429
manifest_bucket_name=None
431430
) -> Tuple[str, bool]:
432431
""" Handle the maven product release tarball deletion process.
@@ -458,10 +457,10 @@ def handle_maven_del(
458457
logger.debug("Valid poms: %s", valid_poms)
459458
succeeded = True
460459
for target in targets:
461-
prefix_ = remove_prefix(target[1], "/")
460+
prefix_ = remove_prefix(target[2], "/")
462461
logger.info("Start deleting files from s3")
463462
s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
464-
bucket = target[0]
463+
bucket = target[1]
465464
(_, failed_files) = s3_client.delete_files(
466465
valid_mvn_paths,
467466
target=(bucket, prefix_),
@@ -471,6 +470,7 @@ def handle_maven_del(
471470
logger.info("Files deletion done\n")
472471

473472
# 4. Delete related manifest from s3
473+
manifest_folder = target[0]
474474
logger.info("Start deleting manifest from s3")
475475
s3_client.delete_manifest(prod_key, manifest_folder, manifest_bucket_name)
476476
logger.info("Manifest deletion is done\n")

‎tests/test_manifest_del.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,9 @@ def test_maven_manifest_delete(self):
4141
product = "commons-client-4.5.6"
4242
handle_maven_del(
4343
test_zip, product,
44-
targets=[(TEST_BUCKET, None)],
44+
targets=[(TEST_TARGET, TEST_BUCKET, None)],
4545
dir_=self.tempdir,
4646
do_index=False,
47-
manifest_folder=TEST_TARGET,
4847
manifest_bucket_name=TEST_MANIFEST_BUCKET
4948
)
5049
uploaded_manifest = list(self.test_manifest_bucket.objects.all())
@@ -78,10 +77,9 @@ def __prepare_maven_content(self):
7877
product = "commons-client-4.5.6"
7978
handle_maven_uploading(
8079
test_zip, product,
81-
targets=[(TEST_BUCKET, None)],
80+
targets=[(TEST_TARGET, TEST_BUCKET, None)],
8281
dir_=self.tempdir,
8382
do_index=False,
84-
manifest_folder=TEST_TARGET,
8583
manifest_bucket_name=TEST_MANIFEST_BUCKET
8684
)
8785

‎tests/test_manifest_upload.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,9 @@ def test_maven_manifest_upload(self):
3535
product = "commons-client-4.5.6"
3636
handle_maven_uploading(
3737
test_zip, product,
38-
targets=[(TEST_BUCKET, None)],
38+
targets=[(TEST_TARGET, TEST_BUCKET, None)],
3939
dir_=self.tempdir,
4040
do_index=False,
41-
manifest_folder=TEST_TARGET,
4241
manifest_bucket_name=TEST_MANIFEST_BUCKET
4342
)
4443

‎tests/test_maven_del.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ def test_ignore_del(self):
5353
handle_maven_del(
5454
test_zip, product_456,
5555
ignore_patterns=[".*.sha1"],
56-
targets=[(TEST_BUCKET, None)],
56+
targets=[(None, TEST_BUCKET, None)],
5757
dir_=self.tempdir,
5858
do_index=False
5959
)
@@ -102,7 +102,7 @@ def __test_prefix_deletion(self, prefix: str):
102102
product_456 = "commons-client-4.5.6"
103103
handle_maven_del(
104104
test_zip, product_456,
105-
targets=[(TEST_BUCKET, prefix)],
105+
targets=[(None, TEST_BUCKET, prefix)],
106106
dir_=self.tempdir, do_index=False
107107
)
108108

@@ -176,7 +176,7 @@ def __test_prefix_deletion(self, prefix: str):
176176
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
177177
handle_maven_del(
178178
test_zip, product_459,
179-
targets=[(TEST_BUCKET, prefix)],
179+
targets=[(None, TEST_BUCKET, prefix)],
180180
dir_=self.tempdir,
181181
do_index=False
182182
)
@@ -189,7 +189,7 @@ def __prepare_content(self, prefix=None):
189189
product_456 = "commons-client-4.5.6"
190190
handle_maven_uploading(
191191
test_zip, product_456,
192-
targets=[(TEST_BUCKET, prefix)],
192+
targets=[(None, TEST_BUCKET, prefix)],
193193
dir_=self.tempdir,
194194
do_index=False
195195
)
@@ -198,7 +198,7 @@ def __prepare_content(self, prefix=None):
198198
product_459 = "commons-client-4.5.9"
199199
handle_maven_uploading(
200200
test_zip, product_459,
201-
targets=[(TEST_BUCKET, prefix)],
201+
targets=[(None, TEST_BUCKET, prefix)],
202202
dir_=self.tempdir,
203203
do_index=False
204204
)

‎tests/test_maven_index.py‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def test_uploading_index(self):
3535
product = "commons-client-4.5.6"
3636
handle_maven_uploading(
3737
test_zip, product,
38-
targets=[(TEST_BUCKET, None)],
38+
targets=[(None, TEST_BUCKET, None)],
3939
dir_=self.tempdir
4040
)
4141

@@ -79,15 +79,15 @@ def test_overlap_upload_index(self):
7979
product_456 = "commons-client-4.5.6"
8080
handle_maven_uploading(
8181
test_zip, product_456,
82-
targets=[(TEST_BUCKET, None)],
82+
targets=[(None, TEST_BUCKET, None)],
8383
dir_=self.tempdir
8484
)
8585

8686
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
8787
product_459 = "commons-client-4.5.9"
8888
handle_maven_uploading(
8989
test_zip, product_459,
90-
targets=[(TEST_BUCKET, None)],
90+
targets=[(None, TEST_BUCKET, None)],
9191
dir_=self.tempdir
9292
)
9393

@@ -139,7 +139,7 @@ def __test_upload_index_with_prefix(self, prefix: str):
139139
product = "commons-client-4.5.6"
140140
handle_maven_uploading(
141141
test_zip, product,
142-
targets=[(TEST_BUCKET, prefix)],
142+
targets=[(None, TEST_BUCKET, prefix)],
143143
dir_=self.tempdir
144144
)
145145

@@ -191,7 +191,7 @@ def test_deletion_index(self):
191191
product_456 = "commons-client-4.5.6"
192192
handle_maven_del(
193193
test_zip, product_456,
194-
targets=[(TEST_BUCKET, None)],
194+
targets=[(None, TEST_BUCKET, None)],
195195
dir_=self.tempdir
196196
)
197197

@@ -239,7 +239,7 @@ def test_deletion_index(self):
239239
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
240240
handle_maven_del(
241241
test_zip, product_459,
242-
targets=[(TEST_BUCKET, None)],
242+
targets=[(None, TEST_BUCKET, None)],
243243
dir_=self.tempdir
244244
)
245245

@@ -262,7 +262,7 @@ def __test_deletion_index_with_prefix(self, prefix: str):
262262
product_456 = "commons-client-4.5.6"
263263
handle_maven_del(
264264
test_zip, product_456,
265-
targets=[(TEST_BUCKET, prefix)],
265+
targets=[(None, TEST_BUCKET, prefix)],
266266
dir_=self.tempdir
267267
)
268268

@@ -309,7 +309,7 @@ def __test_deletion_index_with_prefix(self, prefix: str):
309309
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
310310
handle_maven_del(
311311
test_zip, product_459,
312-
targets=[(TEST_BUCKET, prefix)],
312+
targets=[(None, TEST_BUCKET, prefix)],
313313
dir_=self.tempdir
314314
)
315315

@@ -321,14 +321,14 @@ def __prepare_content(self, prefix=None):
321321
product_456 = "commons-client-4.5.6"
322322
handle_maven_uploading(
323323
test_zip, product_456,
324-
targets=[(TEST_BUCKET, prefix)],
324+
targets=[(None, TEST_BUCKET, prefix)],
325325
dir_=self.tempdir
326326
)
327327

328328
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
329329
product_459 = "commons-client-4.5.9"
330330
handle_maven_uploading(
331331
test_zip, product_459,
332-
targets=[(TEST_BUCKET, prefix)],
332+
targets=[(None, TEST_BUCKET, prefix)],
333333
dir_=self.tempdir
334334
)

‎tests/test_maven_upload.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,15 @@ def test_overlap_upload(self):
4646
product_456 = "commons-client-4.5.6"
4747
handle_maven_uploading(
4848
test_zip, product_456,
49-
targets=[(TEST_BUCKET, None)],
49+
targets=[(None, TEST_BUCKET, None)],
5050
dir_=self.tempdir, do_index=False
5151
)
5252

5353
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
5454
product_459 = "commons-client-4.5.9"
5555
handle_maven_uploading(
5656
test_zip, product_459,
57-
targets=[(TEST_BUCKET, None)],
57+
targets=[(None, TEST_BUCKET, None)],
5858
dir_=self.tempdir, do_index=False
5959
)
6060

@@ -113,7 +113,7 @@ def test_ignore_upload(self):
113113
product_456 = "commons-client-4.5.6"
114114
handle_maven_uploading(
115115
test_zip, product_456, [".*.sha1"],
116-
targets=[(TEST_BUCKET, None)],
116+
targets=[(None, TEST_BUCKET, None)],
117117
dir_=self.tempdir, do_index=False
118118
)
119119

@@ -142,7 +142,7 @@ def __test_prefix_upload(self, prefix: str):
142142
product = "commons-client-4.5.6"
143143
handle_maven_uploading(
144144
test_zip, product,
145-
targets=[(TEST_BUCKET, prefix)],
145+
targets=[(None, TEST_BUCKET, prefix)],
146146
dir_=self.tempdir,
147147
do_index=False
148148
)

‎tests/test_pkgs_dryrun.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def test_maven_upload_dry_run(self):
2828
product = "commons-client-4.5.6"
2929
handle_maven_uploading(
3030
test_zip, product,
31-
targets=[(TEST_BUCKET, None)],
31+
targets=[(None, TEST_BUCKET, None)],
3232
dir_=self.tempdir,
3333
dry_run=True
3434
)
@@ -44,7 +44,7 @@ def test_maven_delete_dry_run(self):
4444
product_456 = "commons-client-4.5.6"
4545
handle_maven_del(
4646
test_zip, product_456,
47-
targets=[(TEST_BUCKET, None)],
47+
targets=[(None, TEST_BUCKET, None)],
4848
dir_=self.tempdir,
4949
dry_run=True
5050
)
@@ -88,15 +88,15 @@ def __prepare_maven_content(self):
8888
product_456 = "commons-client-4.5.6"
8989
handle_maven_uploading(
9090
test_zip, product_456,
91-
targets=[(TEST_BUCKET, None)],
91+
targets=[(None, TEST_BUCKET, None)],
9292
dir_=self.tempdir
9393
)
9494

9595
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.9.zip")
9696
product_459 = "commons-client-4.5.9"
9797
handle_maven_uploading(
9898
test_zip, product_459,
99-
targets=[(TEST_BUCKET, None)],
99+
targets=[(None, TEST_BUCKET, None)],
100100
dir_=self.tempdir
101101
)
102102

0 commit comments

Comments
 (0)