Skip to content

Commit 2541712

Browse files
committed
overrides: Pin kdump-utils-1.0.48-1.fc42 for ppc64le
Reorganized the loop structure to iterate over builds first, followed by arch and then actions, optimizing the number of meta.json downloads and reducing redundancy.
1 parent 7550a58 commit 2541712

File tree

1 file changed

+57
-54
lines changed

1 file changed

+57
-54
lines changed

src/cmd-cloud-prune

Lines changed: 57 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -108,61 +108,64 @@ def main():
108108
builds_json_data = json.load(f)
109109
# Original list of builds
110110
builds = builds_json_data["builds"]
111-
112-
# Prune builds based on the policy
113-
for action in ['cloud-uploads', 'images', 'build']:
114-
if action not in policy[stream]:
115-
continue
116-
duration = convert_duration_to_days(policy[stream][action])
117-
ref_date = today_date - relativedelta(days=int(duration))
118-
pruned_build_ids = []
119-
images_to_keep = policy.get(stream, {}).get("images-keep", [])
120-
121-
print(f"Pruning resources of type {action} older than {policy[stream][action]} ({ref_date.date()}) on stream {stream}")
122-
# Enumerating in reverse to go from the oldest build to the newest one
123-
for build in reversed(builds):
124-
build_id = build["id"]
125-
(build_date, _) = parse_fcos_version_to_timestamp_and_stream(build_id)
126-
if build_date >= ref_date:
127-
break
128-
129-
previous_cleanup = build.get("policy-cleanup", {})
130-
if action in previous_cleanup:
131-
# If we are in here then there has been some previous cleanup of
132-
# this type run for this build. For all types except `images` we
133-
# can just continue.
134-
if action != "images":
135-
print(f"Build {build_id} has already had {action} pruning completed")
111+
pruned_build_ids = []
112+
113+
# Iterate through builds from oldest to newest
114+
for build in reversed(builds):
115+
build_id = build["id"]
116+
(build_date, _) = parse_fcos_version_to_timestamp_and_stream(build_id)
117+
118+
# For each build, iterate over arches first to minimize downloads of meta.json per arch
119+
for arch in build["arches"]:
120+
print(f"Processing {arch} for build {build_id}")
121+
meta_prefix = os.path.join(prefix, f"{build_id}/{arch}/meta.json")
122+
meta_json = get_json_from_s3(s3_client, bucket, meta_prefix) # Download meta.json once per arch
123+
images = get_supported_images(meta_json)
124+
current_build = Build(id=build_id, images=images, arch=arch, meta_json=meta_json)
125+
126+
# Iterate over actions (policy types) to apply pruning
127+
for action in ['cloud-uploads', 'images', 'build']:
128+
if action not in policy[stream]:
136129
continue
137-
else:
138-
# OK `images` has been pruned before, but we need to check
139-
# that all the images were pruned that match the current policy.
140-
# i.e. there may be additional images we need prune
141-
previous_images_kept = previous_cleanup.get("images-kept", [])
142-
if set(images_to_keep) == set(previous_images_kept):
143-
print(f"Build {build_id} has already had {action} pruning completed")
144-
continue
145-
146-
for arch in build["arches"]:
147-
print(f"Pruning {arch} {action} for {build_id}")
148-
meta_prefix = os.path.join(prefix, f"{build_id}/{arch}/meta.json")
149-
meta_json = get_json_from_s3(s3_client, bucket, meta_prefix)
150-
# Make sure the meta.json doesn't contain any cloud_platform that is not supported for pruning yet.
151-
images = get_supported_images(meta_json)
152-
current_build = Build(id=build_id, images=images, arch=arch, meta_json=meta_json)
153-
154-
match action:
155-
case "cloud-uploads":
156-
prune_cloud_uploads(current_build, cloud_config, args.dry_run)
157-
# Prune through images that are not mentioned in images-keep
158-
case "images":
159-
prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix)
160-
# Fully prune releases that are very old including deleting the directory in s3 for that build.
161-
case "build":
162-
prune_build(s3_client, bucket, prefix, build_id, args.dry_run)
163-
pruned_build_ids.append(build_id)
164-
# Update policy-cleanup after processing all arches for the build
165-
policy_cleanup = build.setdefault("policy-cleanup", {})
130+
action_duration = convert_duration_to_days(policy[stream][action])
131+
ref_date = today_date - relativedelta(days=int(action_duration))
132+
images_to_keep = policy.get(stream, {}).get("images-keep", [])
133+
134+
# Check if build date is beyond the reference date
135+
if build_date < ref_date:
136+
previous_cleanup = build.get("policy-cleanup", {})
137+
138+
# Skip if the action has been handled previously for the build
139+
if action in previous_cleanup:
140+
# If we are in here then there has been some previous cleanup of
141+
# this type run for this build. For all types except `images` we
142+
# can just continue.
143+
if action != "images":
144+
print(f"Build {build_id} has already had {action} pruning completed")
145+
continue
146+
# OK `images` has been pruned before, but we need to check
147+
# that all the images were pruned that match the current policy.
148+
# i.e. there may be additional images we need prune
149+
elif set(images_to_keep) == set(previous_cleanup.get("images-kept", [])):
150+
print(f"Build {build_id} has already had {action} pruning completed")
151+
continue
152+
153+
# Pruning actions based on type
154+
print(f"Pruning {arch} {action} for {build_id}")
155+
match action:
156+
case "cloud-uploads":
157+
prune_cloud_uploads(current_build, cloud_config, args.dry_run)
158+
# Prune through images that are not mentioned in images-keep
159+
case "images":
160+
prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix)
161+
# Fully prune releases that are very old including deleting the directory in s3 for that build.
162+
case "build":
163+
prune_build(s3_client, bucket, prefix, build_id, args.dry_run)
164+
pruned_build_ids.append(build_id)
165+
166+
# Update policy-cleanup after pruning actions for the architecture
167+
policy_cleanup = build.setdefault("policy-cleanup", {})
168+
for action in policy[stream].keys(): # Only update actions specified in policy[stream]
166169
match action:
167170
case "cloud-uploads":
168171
if "cloud-uploads" not in policy_cleanup:

0 commit comments

Comments
 (0)