Skip to content

Commit 49e0556

Browse files
rajulkumarrajulkumar
authored
Merge pull request #327 from rajulkumar/reduce_copy_repo_calls
copy-repo makes only two copy calls for a repo pair [RHELDST-28235]
2 parents fa6b407 + 1788ebd commit 49e0556

File tree

6 files changed

+139
-50
lines changed

6 files changed

+139
-50
lines changed

‎src/pubtools/_pulp/tasks/copy_repo.py‎

Lines changed: 71 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from itertools import chain
55

66
import attr
7-
from more_executors.futures import f_map, f_sequence, f_proxy
7+
from more_executors.futures import f_map, f_sequence, f_proxy, f_flat_map, f_return
88
from pubtools.pulplib import (
99
ContainerImageRepository,
1010
Criteria,
@@ -73,6 +73,23 @@
7373
ContentType(("package_langpacks",)),
7474
)
7575

76+
RPM_CONTENT_TYPES = (
77+
"rpm",
78+
"srpm",
79+
)
80+
81+
NON_RPM_CONTENT_TYPES = (
82+
"iso",
83+
"erratum",
84+
"modulemd",
85+
"modulemd_defaults",
86+
"yum_repo_metadata_file",
87+
"package_group",
88+
"package_category",
89+
"package_environment",
90+
"package_langpacks",
91+
)
92+
7693

7794
@attr.s(slots=True)
7895
class RepoCopy(object):
@@ -93,38 +110,56 @@ def content_type_criteria(self):
93110
out = None
94111

95112
def str_to_content_type(content_type_id):
96-
out = None
97113
for item in CONTENT_TYPES:
98114
if content_type_id in item.content_type_ids:
99-
out = item
100-
break
101-
102-
if out is None:
103-
self.fail("Unsupported content type: %s", content_type_id)
104-
105-
return out
115+
return item
106116

107117
if self.args.content_type:
108118
# replace srpm with rpm - we don't need to specify it separately and remove duplicated entries
109119
content_types = set(
110-
map(lambda x: x.replace("srpm", "rpm"), self.args.content_type)
120+
map(
121+
lambda x: x.lower().strip().replace("srpm", "rpm"),
122+
self.args.content_type,
123+
)
124+
)
125+
126+
# check for unsupported content types
127+
unsupported = content_types.difference(
128+
RPM_CONTENT_TYPES + NON_RPM_CONTENT_TYPES
111129
)
112-
content_types = [
113-
str_to_content_type(t.lower().strip()) for t in content_types
130+
if unsupported:
131+
self.fail("Unsupported content type(s): %s", ",".join(unsupported))
132+
133+
rpm_content_types = [
134+
str_to_content_type(t) for t in content_types if t in RPM_CONTENT_TYPES
135+
]
136+
non_rpm_content_types = [
137+
t for t in content_types if t in NON_RPM_CONTENT_TYPES
114138
]
139+
140+
# NOTE: Order of appending the criteria is critical here.
141+
# Non-rpm content types should be copied first as it may contain modulemd
142+
# content type. Modulemd units should be copied before the modular rpms,
143+
# so as in case of a failure or partial copy, the modular rpms aren't
144+
# available to the users. Hence, non-rpm content type criteria is appended
145+
# first in the list of criteria
115146
criteria = []
116-
in_matcher = [] # to aggregate content types for Criteria.with_field()
117-
118-
for item in sorted(content_types):
119-
if item.klass:
120-
criteria.append(
121-
Criteria.with_unit_type(item.klass, unit_fields=item.fields)
122-
)
123-
else:
124-
in_matcher.extend(item.content_type_ids)
125-
if in_matcher:
147+
148+
# criteria for all non-rpm content types
149+
# unit_fields are ignored as they are small in size and the repos have
150+
# small unit counts for non-rpm content types
151+
criteria.append(
152+
Criteria.with_field(
153+
"content_type_id", Matcher.in_(sorted(non_rpm_content_types))
154+
)
155+
)
156+
157+
# criteria for rpm content types
158+
# unit_fields to keep a check on memory consumption with large rpm unit
159+
# counts in the repo
160+
for item in sorted(rpm_content_types):
126161
criteria.append(
127-
Criteria.with_field("content_type_id", Matcher.in_(in_matcher))
162+
Criteria.with_unit_type(item.klass, unit_fields=item.fields)
128163
)
129164

130165
out = criteria
@@ -214,14 +249,23 @@ def repo_copy(copy_tasks, repo):
214249
tasks = list(chain.from_iterable(copy_tasks))
215250
return RepoCopy(tasks=tasks, repo=repo)
216251

217-
for src_repo, dest_repo in repo_pairs:
252+
for src_repo, dest_repo in sorted(repo_pairs):
218253
one_pair_copies = []
254+
tasks_f = f_return()
219255
for item in criteria or [None]:
220-
tasks_f = self.pulp_client.copy_content(
221-
src_repo, dest_repo, criteria=item
256+
# ensure the criterias are processed and completed/resolved in order
257+
# so that non-rpm copy completes before rpm copy
258+
# pylint:disable=cell-var-from-loop
259+
tasks_f = f_flat_map(
260+
tasks_f,
261+
lambda _: self.pulp_client.copy_content(
262+
src_repo,
263+
dest_repo,
264+
criteria=item,
265+
),
222266
)
267+
# pylint:enable=cell-var-from-loop
223268
one_pair_copies.append(tasks_f)
224-
225269
f = f_map(f_sequence(one_pair_copies), partial(repo_copy, repo=dest_repo))
226270
f = f_map(f, self.log_copy)
227271
fts.append(f)

‎tests/copy_repo/test_copy_repo.py‎

Lines changed: 63 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,16 @@ def test_copy_repo_multiple_content_types(command_tester, fake_collector):
385385
relative_url="another/publish/url",
386386
mutable_urls=["repomd.xml"],
387387
)
388+
repoC = YumRepository(
389+
id="other-yumrepo",
390+
relative_url="other/publish/url",
391+
mutable_urls=["repomd.xml"],
392+
)
393+
repoD = YumRepository(
394+
id="yet-another-yumrepo",
395+
relative_url="yet/another/publish/url",
396+
mutable_urls=["repomd.xml"],
397+
)
388398

389399
files = [
390400
RpmUnit(
@@ -400,11 +410,28 @@ def test_copy_repo_multiple_content_types(command_tester, fake_collector):
400410
name="mymod", stream="s1", version=123, context="a1c2", arch="s390x"
401411
),
402412
]
413+
files2 = [
414+
RpmUnit(
415+
name="dash",
416+
version="1.24",
417+
release="1.test8",
418+
arch="x86_64",
419+
sha256sum="a" * 64,
420+
md5sum="b" * 32,
421+
signing_key="aabbcc",
422+
),
423+
ModulemdUnit(
424+
name="othermod", stream="s1", version=123, context="a1c2", arch="s390x"
425+
),
426+
]
403427

404428
with FakeCopyRepo() as task_instance:
405429
task_instance.pulp_client_controller.insert_repository(repoA)
406430
task_instance.pulp_client_controller.insert_repository(repoB)
431+
task_instance.pulp_client_controller.insert_repository(repoC)
432+
task_instance.pulp_client_controller.insert_repository(repoD)
407433
task_instance.pulp_client_controller.insert_units(repoA, files)
434+
task_instance.pulp_client_controller.insert_units(repoC, files2)
408435

409436
# It should run with expected output.
410437
command_tester.test(
@@ -422,6 +449,7 @@ def test_copy_repo_multiple_content_types(command_tester, fake_collector):
422449
"--content-type",
423450
"erratum",
424451
"some-yumrepo,another-yumrepo",
452+
"other-yumrepo,yet-another-yumrepo",
425453
],
426454
)
427455

@@ -441,6 +469,16 @@ def test_copy_repo_multiple_content_types(command_tester, fake_collector):
441469
"signing_key": None,
442470
"build": None,
443471
},
472+
{
473+
"state": "PUSHED",
474+
"origin": "pulp",
475+
"src": None,
476+
"dest": "yet-another-yumrepo",
477+
"filename": "dash-1.24-1.test8.x86_64.rpm",
478+
"checksums": {"sha256": "a" * 64},
479+
"signing_key": None,
480+
"build": None,
481+
},
444482
{
445483
"state": "PUSHED",
446484
"origin": "pulp",
@@ -451,6 +489,16 @@ def test_copy_repo_multiple_content_types(command_tester, fake_collector):
451489
"signing_key": None,
452490
"build": None,
453491
},
492+
{
493+
"state": "PUSHED",
494+
"origin": "pulp",
495+
"src": None,
496+
"dest": "yet-another-yumrepo",
497+
"filename": "othermod:s1:123:a1c2:s390x",
498+
"checksums": None,
499+
"signing_key": None,
500+
"build": None,
501+
},
454502
]
455503

456504

@@ -482,11 +530,11 @@ def test_copy_repo_criteria(command_tester):
482530
"--content-type", # duplicate
483531
"modulemd",
484532
"--content-type",
485-
"iso",
533+
"ISO",
486534
"--content-type",
487-
"erratum",
535+
"Erratum",
488536
"--content-type",
489-
"package_group",
537+
"package_group ",
490538
"--content-type",
491539
"package_langpacks",
492540
"some-yumrepo,another-yumrepo",
@@ -507,6 +555,18 @@ def test_copy_repo_criteria(command_tester):
507555
[
508556
str(item)
509557
for item in [
558+
Criteria.with_field(
559+
"content_type_id",
560+
Matcher.in_(
561+
[
562+
"erratum",
563+
"iso",
564+
"modulemd",
565+
"package_group",
566+
"package_langpacks",
567+
]
568+
),
569+
),
510570
Criteria.with_unit_type(
511571
RpmUnit,
512572
unit_fields=(
@@ -519,22 +579,6 @@ def test_copy_repo_criteria(command_tester):
519579
"signing_key",
520580
),
521581
),
522-
Criteria.with_unit_type(ErratumUnit, unit_fields=("unit_id",)),
523-
Criteria.with_unit_type(
524-
ModulemdUnit,
525-
unit_fields=(
526-
"name",
527-
"stream",
528-
"version",
529-
"context",
530-
"arch",
531-
),
532-
),
533-
Criteria.with_unit_type(FileUnit, unit_fields=("unit_id",)),
534-
Criteria.with_field(
535-
"content_type_id",
536-
Matcher.in_(["package_group", "package_langpacks"]),
537-
),
538582
]
539583
]
540584
)

‎tests/logs/copy_repo/test_copy_repo/test_copy_invalid_content_type.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[ INFO] Check repos: started
22
[ INFO] Check repos: finished
33
[ INFO] Copy content: started
4-
[ ERROR] Unsupported content type: container
4+
[ ERROR] Unsupported content type(s): container
55
[ ERROR] Copy content: failed
66
# Raised: 30

‎tests/logs/copy_repo/test_copy_repo/test_copy_repo_criteria.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[ INFO] Check repos: started
22
[ INFO] Check repos: finished
33
[ INFO] Copy content: started
4-
[ WARNING] another-yumrepo: no content copied, tasks: 23a7711a-8133-2876-37eb-dcd9e87a1613, 82e2e662-f728-b4fa-4248-5e3a0a5d2f34, d4713d60-c8a7-0639-eb11-67b367a9c378, e3e70682-c209-4cac-629f-6fbed82c07cd, e6f4590b-9a16-4106-cf6a-659eb4862b21
4+
[ WARNING] another-yumrepo: no content copied, tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34, e3e70682-c209-4cac-629f-6fbed82c07cd
55
[ INFO] Copy content: finished
66
[ INFO] Record push items: started
77
[ INFO] Record push items: finished

‎tests/logs/copy_repo/test_copy_repo/test_copy_repo_multiple_content_types.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
[ INFO] Check repos: started
22
[ INFO] Check repos: finished
33
[ INFO] Copy content: started
4-
[ INFO] another-yumrepo: copied 1 modulemd(s), 1 rpm(s), tasks: 23a7711a-8133-2876-37eb-dcd9e87a1613, 82e2e662-f728-b4fa-4248-5e3a0a5d2f34, d4713d60-c8a7-0639-eb11-67b367a9c378, e3e70682-c209-4cac-629f-6fbed82c07cd
4+
[ INFO] yet-another-yumrepo: copied 1 modulemd(s), 1 rpm(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34, e3e70682-c209-4cac-629f-6fbed82c07cd
5+
[ INFO] another-yumrepo: copied 1 modulemd(s), 1 rpm(s), tasks: 23a7711a-8133-2876-37eb-dcd9e87a1613, d4713d60-c8a7-0639-eb11-67b367a9c378
56
[ INFO] Copy content: finished
67
[ INFO] Record push items: started
78
[ INFO] Record push items: finished

‎tests/logs/copy_repo/test_copy_repo/test_invalid_content_type.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[ INFO] Check repos: started
22
[ INFO] Check repos: finished
33
[ INFO] Copy content: started
4-
[ ERROR] Unsupported content type: container
4+
[ ERROR] Unsupported content type(s): container
55
[ ERROR] Copy content: failed
66
# Raised: 30

0 commit comments

Comments
 (0)