Skip to content

Commit 48fba72

Browse files
gursewak1997gursewak1997
committed
cmd-cloud-prune: Refactor to iterate by build, arch, then action
Reorganized the loop structure to iterate over builds first, followed by arch and then actions, optimizing the number of meta.json downloads and reducing redundancy.
1 parent c3e1bec commit 48fba72

File tree

1 file changed

+57
-54
lines changed

1 file changed

+57
-54
lines changed

‎src/cmd-cloud-prune‎

Lines changed: 57 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -108,61 +108,64 @@ def main():
108108
builds_json_data = json.load(f)
109109
# Original list of builds
110110
builds = builds_json_data["builds"]
111-
112-
# Prune builds based on the policy
113-
for action in ['cloud-uploads', 'images', 'build']:
114-
if action not in policy[stream]:
115-
continue
116-
duration = convert_duration_to_days(policy[stream][action])
117-
ref_date = today_date - relativedelta(days=int(duration))
118-
pruned_build_ids = []
119-
images_to_keep = policy.get(stream, {}).get("images-keep", [])
120-
121-
print(f"Pruning resources of type {action} older than {policy[stream][action]} ({ref_date.date()}) on stream {stream}")
122-
# Enumerating in reverse to go from the oldest build to the newest one
123-
for build in reversed(builds):
124-
build_id = build["id"]
125-
(build_date, _) = parse_fcos_version_to_timestamp_and_stream(build_id)
126-
if build_date >= ref_date:
127-
break
128-
129-
previous_cleanup = build.get("policy-cleanup", {})
130-
if action in previous_cleanup:
131-
# If we are in here then there has been some previous cleanup of
132-
# this type run for this build. For all types except `images` we
133-
# can just continue.
134-
if action != "images":
135-
print(f"Build {build_id} has already had {action} pruning completed")
111+
pruned_build_ids = []
112+
images_to_keep = policy.get(stream, {}).get("images-keep", [])
113+
114+
# Iterate through builds from oldest to newest
115+
for build in reversed(builds):
116+
build_id = build["id"]
117+
build_date, _ = parse_fcos_version_to_timestamp_and_stream(build_id)
118+
119+
# For each build, iterate over arches first to minimize downloads of meta.json per arch
120+
for arch in build["arches"]:
121+
print(f"Processing {arch} for build {build_id}")
122+
meta_prefix = os.path.join(prefix, f"{build_id}/{arch}/meta.json")
123+
meta_json = get_json_from_s3(s3_client, bucket, meta_prefix) # Download meta.json once per arch
124+
images = get_supported_images(meta_json)
125+
current_build = Build(id=build_id, images=images, arch=arch, meta_json=meta_json)
126+
127+
# Iterate over actions (policy types) to apply pruning
128+
for action in ['cloud-uploads', 'images', 'build']:
129+
if action not in policy[stream]:
136130
continue
137-
else:
138-
# OK `images` has been pruned before, but we need to check
139-
# that all the images were pruned that match the current policy.
140-
# i.e. there may be additional images we need prune
141-
previous_images_kept = previous_cleanup.get("images-kept", [])
142-
if set(images_to_keep) == set(previous_images_kept):
143-
print(f"Build {build_id} has already had {action} pruning completed")
144-
continue
145-
146-
for arch in build["arches"]:
147-
print(f"Pruning {arch} {action} for {build_id}")
148-
meta_prefix = os.path.join(prefix, f"{build_id}/{arch}/meta.json")
149-
meta_json = get_json_from_s3(s3_client, bucket, meta_prefix)
150-
# Make sure the meta.json doesn't contain any cloud_platform that is not supported for pruning yet.
151-
images = get_supported_images(meta_json)
152-
current_build = Build(id=build_id, images=images, arch=arch, meta_json=meta_json)
153-
154-
match action:
155-
case "cloud-uploads":
156-
prune_cloud_uploads(current_build, cloud_config, args.dry_run)
157-
# Prune through images that are not mentioned in images-keep
158-
case "images":
159-
prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix)
160-
# Fully prune releases that are very old including deleting the directory in s3 for that build.
161-
case "build":
162-
prune_build(s3_client, bucket, prefix, build_id, args.dry_run)
163-
pruned_build_ids.append(build_id)
164-
# Update policy-cleanup after processing all arches for the build
165-
policy_cleanup = build.setdefault("policy-cleanup", {})
131+
action_duration = convert_duration_to_days(policy[stream][action])
132+
ref_date = today_date - relativedelta(days=int(action_duration))
133+
134+
# Check if build date is beyond the reference date
135+
if build_date < ref_date:
136+
previous_cleanup = build.get("policy-cleanup", {})
137+
138+
# Skip if the action has been handled previously for the build
139+
if action in previous_cleanup:
140+
# If we are in here then there has been some previous cleanup of
141+
# this type run for this build. For all types except `images` we
142+
# can just continue.
143+
if action != "images":
144+
print(f"Build {build_id} has already had {action} pruning completed")
145+
continue
146+
# OK `images` has been pruned before, but we need to check
147+
# that all the images were pruned that match the current policy.
148+
# i.e. there may be additional images we need prune
149+
elif set(images_to_keep) == set(previous_cleanup.get("images-kept", [])):
150+
print(f"Build {build_id} has already had {action} pruning completed")
151+
continue
152+
153+
# Pruning actions based on type
154+
print(f"Pruning {arch} {action} for {build_id}")
155+
match action:
156+
case "cloud-uploads":
157+
prune_cloud_uploads(current_build, cloud_config, args.dry_run)
158+
# Prune through images that are not mentioned in images-keep
159+
case "images":
160+
prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix)
161+
# Fully prune releases that are very old including deleting the directory in s3 for that build.
162+
case "build":
163+
prune_build(s3_client, bucket, prefix, build_id, args.dry_run)
164+
pruned_build_ids.append(build_id)
165+
166+
# Update policy-cleanup after pruning actions for the architecture
167+
policy_cleanup = build.setdefault("policy-cleanup", {})
168+
for action in policy[stream].keys(): # Only update actions specified in policy[stream]
166169
match action:
167170
case "cloud-uploads":
168171
if "cloud-uploads" not in policy_cleanup:

0 commit comments

Comments
 (0)