11#!/usr/bin/python3
22#
3+ # in default mode:
34# make a combined "manifest-list" container out of two arch-specific containers
45# searches for latest tags on HOST/{AMD,ARM}64_REPO, makes sure they refer
56# to the same Ceph SHA1, and creates a manifest-list ("fat") image on
6- # MANIFEST_HOST/MANIFEST_REPO with the 'standard' set of tags.
7+ # MANIFEST_HOST/MANIFEST_REPO with the 'standard' set of tags:
8+ # v<major>
9+ # v<major>.<minor>
10+ # v<major>.<minor>.<micro>
11+ # v<major>.<minor>.<micro>-<YYYYMMDD>
712#
8- # uses scratch local manifest LOCALMANIFEST, will be destroyed if present
13+ # uses scratch local manifest LOCALMANIFEST, defined here; will be destroyed if present
14+ #
15+ # in promote mode (by adding the --promote argument):
16+ # instead of building the manifest-list container, copy it
17+ # (and all of its tags) from the prerelease repo to the release repo
18+ #
19+ # Assumes valid logins to the necessary hosts/repos with permission to write images
20+ #
21+ # Environment variables to set:
22+ # ARCH_SPECIFIC_HOST (default 'quay.ceph.io'): host of prerelease repos
23+ # AMD64_REPO (default 'ceph/prerelease-amd64') prerelease amd64 repo
24+ # ARM64_REPO (default 'ceph/prerelease-arm64') prerelease arm64 repo
25+ # MANIFEST_HOST (default 'quay.ceph.io') prerelease manifest-list host
26+ # MANIFEST_REPO (default 'ceph/prerelease') prerelease manifest-list repo
27+ # RELEASE_MANIFEST_HOST (default 'quay.io') release host
28+ # RELEASE_MANIFEST_REPO (default 'ceph/ceph') release repo
929
30+
31+ import argparse
1032from datetime import datetime
1133import functools
1234import json
1537import subprocess
1638import sys
1739
18- # optional env vars (will default if not set)
19-
20- OPTIONAL_VARS = (
21- 'HOST' ,
22- 'AMD64_REPO' ,
23- 'ARM64_REPO' ,
24- 'MANIFEST_HOST' ,
25- 'MANIFEST_REPO' ,
26- )
27-
2840# Manifest image. Will be destroyed if already present.
2941LOCALMANIFEST = 'localhost/m'
3042
@@ -47,10 +59,7 @@ def run_command(args):
4759 return True , result .stdout , result .stderr
4860
4961 except subprocess .CalledProcessError as e :
50- print (f"Command '{ e .cmd } { e .returncode } )
51- print ("Error output:" )
52- print (e .stderr )
53- return False , result .stdout , result .stderr
62+ return False , e .output , e .stderr
5463
5564
5665def get_command_output (args ):
@@ -68,10 +77,16 @@ def run_command_show_failure(args):
6877
6978
7079@functools .lru_cache
80+ def get_tags (path ):
81+ cmdout = get_command_output (f'skopeo list-tags docker://{ path } )
82+ return json .loads (cmdout )['Tags' ]
83+
84+
7185def get_latest_tag (path ):
72- latest_tag = json .loads (
73- get_command_output (f'skopeo list-tags docker://{ path } )
74- )['Tags' ][- 1 ]
86+ try :
87+ latest_tag = get_tags (path )[- 1 ]
88+ except IndexError :
89+ return None
7590 return latest_tag
7691
7792
@@ -84,27 +99,53 @@ def get_image_inspect(path):
8499
85100
86101def get_sha1 (info ):
87- return info ['Labels' ]['GIT_COMMIT' ]
102+ labels = info .get ('Labels' , None )
103+ if not labels :
104+ return None
105+ return labels .get ('CEPH_SHA1' , None )
88106
89107
90- def main ():
91- host = os .environ .get ('HOST' , 'quay.io' )
92- amd64_repo = os .environ .get ('AMD64_REPO' , 'ceph/ceph-amd64' )
93- arm64_repo = os .environ .get ('ARM64_REPO' , 'ceph/ceph-arm64' )
94- manifest_host = os .environ .get ('MANIFEST_HOST' , host )
95- manifest_repo = os .environ .get ('MANIFEST_REPO' , 'ceph/ceph' )
108+ @functools .lru_cache
109+ def get_all_matching_digest_tags (path , tag ):
110+
111+ matching_tags = list ()
112+ digest = get_image_inspect (f'{ path } { tag } )['Digest' ]
113+
114+ for t in get_tags (path ):
115+ this_digest = get_image_inspect (f'{ path } { t } )['Digest' ]
116+ if this_digest == digest :
117+ matching_tags .append (t )
118+
119+ return matching_tags
120+
121+
122+ def parse_args ():
123+ ap = argparse .ArgumentParser ()
124+ ap .add_argument ('-n' , '--dry-run' , action = 'store_true' , help = 'do all local manipulations but do not push final containers to MANIFEST_HOST, or in --promote, calculate but do not copy images to release host' )
125+ ap .add_argument ('-P' , '--promote' , action = 'store_true' , help = 'promote newest prerelease manifest container to released (move from MANIFEST_HOST to RELEASE_MANIFEST_HOST' )
126+ args = ap .parse_args ()
127+ return args
128+
129+ def build_prerelease (sysargs ):
130+ global args
131+
132+ arch_specific_host = os .environ .get ('ARCH_SPECIFIC_HOST' , 'quay.ceph.io' )
133+ amd64_repo = os .environ .get ('AMD64_REPO' , 'ceph/prerelease-amd64' )
134+ arm64_repo = os .environ .get ('ARM64_REPO' , 'ceph/prerelease-arm64' )
135+ manifest_host = os .environ .get ('MANIFEST_HOST' , 'quay.ceph.io' )
136+ manifest_repo = os .environ .get ('MANIFEST_REPO' , 'ceph/prerelease' )
137+
96138 dump_vars (
97- ('host ' ,
139+ ('arch_specific_host ' ,
98140 'amd64_repo' ,
99141 'arm64_repo' ,
100142 'manifest_host' ,
101143 'manifest_repo' ,
102144 ),
103145 locals ())
104-
105146 repopaths = (
106- f'{ host } { amd64_repo } ,
107- f'{ host } { arm64_repo } ,
147+ f'{ arch_specific_host } { amd64_repo } ,
148+ f'{ arch_specific_host } { arm64_repo } ,
108149 )
109150 tags = [get_latest_tag (p ) for p in repopaths ]
110151 print (f'latest tags: amd64:{ tags [0 ]} { tags [1 ]} )
@@ -145,8 +186,8 @@ def main():
145186
146187 # create manifest list image with the standard list of tags
147188 # ignore failure on manifest rm
148- run_command (f'podman manifest rm localhost/m ' )
149- run_command_show_failure (f'podman manifest create localhost/m ' )
189+ run_command (f'podman manifest rm { LOCALMANIFEST } )
190+ run_command_show_failure (f'podman manifest create { LOCALMANIFEST } )
150191 for p in paths_with_tags :
151192 run_command_show_failure (f'podman manifest add m { p } )
152193 base = f'{ manifest_host } { manifest_repo }
@@ -156,8 +197,55 @@ def main():
156197 f'v{ major } { minor } { micro } ,
157198 f'v{ major } { minor } { micro } { datetime .today ().strftime ("%Y%m%d" )} ,
158199 ):
159- run_command_show_failure (
160- f'podman manifest push localhost/m { base } { t } )
200+ if sysargs .dry_run :
201+ print (f'skipping podman manifest push { LOCALMANIFEST } { base } { t } )
202+ else :
203+ run_command_show_failure (
204+ f'podman manifest push { LOCALMANIFEST } { base } { t } )
205+
206+ def promote (sysargs ):
207+ manifest_host = os .environ .get ('MANIFEST_HOST' , 'quay.ceph.io' )
208+ manifest_repo = os .environ .get ('MANIFEST_REPO' , 'ceph/prerelease' )
209+ release_manifest_host = os .environ .get ('RELEASE_MANIFEST_HOST' , 'quay.io' )
210+ release_manifest_repo = os .environ .get ('RELEASE_MANIFEST_REPO' , 'ceph/ceph' )
211+ dump_vars (
212+ ('manifest_host' ,
213+ 'manifest_repo' ,
214+ 'release_manifest_host' ,
215+ 'release_manifest_repo' ,
216+ ),
217+ locals ())
218+
219+ manifest_path = f'{ manifest_host } { manifest_repo }
220+ release_path = f'{ release_manifest_host } { release_manifest_repo }
221+ latest_tag = get_latest_tag (manifest_path )
222+ all_tags = get_all_matching_digest_tags (manifest_path , latest_tag )
223+
224+ copypaths = list ()
225+ for t in all_tags :
226+ from_path = f'{ manifest_path } { t }
227+ to_path = f'{ release_path } { t }
228+ copypaths .append ((from_path , to_path ))
229+
230+ if sysargs .dry_run :
231+ for f , t in copypaths :
232+ print (f'dry-run: Would copy: { f } { t } )
233+ return (0 )
234+
235+ for f , t in copypaths :
236+ print (f'Will copy: { f } { t } )
237+
238+ for f , t in copypaths :
239+ run_command_show_failure (f'skopeo copy --multi-arch=all docker://{ f } { t } )
240+
241+
242+ def main ():
243+ args = parse_args ()
244+
245+ if args .promote :
246+ promote (args )
247+ else :
248+ build_prerelease (args )
161249
162250
163251if (__name__ == '__main__' ):
0 commit comments