cmd-build-with-buildah: enhance version handling, add yumrepos support, add autolocking

src/cmd-build-with-buildah

@@ -14,15 +14,21 @@ Usage: coreos-assembler build-with-buildah
   `cosa build` will pivot to this script when the environment variable `COREOS_ASSEMBLER_BUILD_WITH_BUILDAH` is set.
 
   The following options are supported:
-  --version           Use the versionary script from the source config to drive version.
+  --version=VERSION   Use the given version instead of using versionary.
+  --versionary        Generate non-development version using versionary.
   --direct            Run buildah directly rather than within supermin.
+  --autolock=VERSION  If no base lockfile used, create one from any arch build of `VERSION`.
+                      Note this is automatically enabled when adding to an existing multi-arch
+                      non-strict build.
 EOF
 }
 
 VERSION=
+VERSIONARY=
 DIRECT=
+AUTOLOCK_VERSION=
 rc=0
-options=$(getopt --options h,v,d --longoptions help,version:,direct -- "$@") || rc=$?
+options=$(getopt --options h,d --longoptions help,version:,versionary,direct,autolock: -- "$@") || rc=$?
 [ $rc -eq 0 ] || {
     print_help
     exit 1
@@ -34,13 +40,20 @@ while true; do
             print_help
             exit 0
             ;;
-        -v | --version)
+        --version)
             shift
             VERSION=$1
             ;;
+        --versionary)
+            VERSIONARY=1
+            ;;
         -d | --direct)
             DIRECT=1
             ;;
+        --autolock)
+            shift;
+            AUTOLOCK_VERSION=$1
+            ;;
         --)
             shift
             break
@@ -57,7 +70,11 @@ done
 
 if [ -z "${VERSION}" ]; then
     # let error out if file does not exist
-    VERSION=$(src/config/versionary)
+    if [ -z "${VERSIONARY}" ]; then
+        VERSION=$(src/config/versionary --dev)
+    else
+        VERSION=$(src/config/versionary)
+    fi
 fi
 
 build_with_buildah() {
@@ -70,21 +87,52 @@ build_with_buildah() {
 
     tmp_oci_archive_path=$(realpath "${tempdir}/out.ociarchive")
 
-    argsfile=build-args.conf
-    if [ -n "${variant:-}" ]; then
+    initconfig="src/config.json"
+    if [ -f "${initconfig}" ]; then
+        variant="$(jq --raw-output '."coreos-assembler.config-variant"' "${initconfig}")"
+        manifest="src/config/manifest-${variant}.yaml"
         argsfile=build-args-${variant}.conf
+    else
+        manifest="src/config/manifest.yaml"
+        argsfile=build-args.conf
     fi
 
-    if [ "$(check_build_exists "${VERSION}")" == "True" ]; then
+    if [ -e "builds/$VERSION" ]; then
         echo "Build ${VERSION} already exists"
         exit 0
     fi
 
+    # Apply autolock from another build for this version (or for another version if
+    # explicitly provided via --autolock) if no base lockfile exists.
+    lockfile="manifest-lock.${arch}.json"
+    if [ ! -f "src/config/${lockfile}" ] && { [ -n "${VERSION}" ] || [ -n "${AUTOLOCK_VERSION}" ]; }; then
+        autolockfile=$(tmprepo=tmp/repo; workdir=.;
+                       generate_autolock "${AUTOLOCK_VERSION:-${VERSION}}")
+        if [ -n "${autolockfile}" ]; then
+            echo "Injecting autolock-generated ${lockfile}..."
+            cp "${autolockfile}" "${tempdir}/src/${lockfile}"
+        fi
+    fi
+
     set -- build --security-opt=label=disable --cap-add=all --device /dev/fuse \
             --build-arg-file "$argsfile" -v "$(realpath "${tempdir}/src")":/run/src \
             --build-arg VERSION="${VERSION}" \
             -t oci-archive:"${tmp_oci_archive_path}"
 
+    # XXX: Temporary hack until we have https://github.com/coreos/rpm-ostree/pull/5454
+    # which would allow us to fold this back into the build process.
+    # shellcheck source=/dev/null
+    stream=$(yaml2json "$manifest" /dev/stdout | jq -r '.variables.stream')
+    if [ "${stream}" != null ]; then
+        set -- "$@" --label fedora-coreos.stream="$stream"
+    fi
+
+    if [ -d "src/yumrepos" ] && [ -e "src/yumrepos/${variant:-}.repo" ]; then
+        set -- "$@" --secret id=yumrepos,src="$(realpath "src/yumrepos/$variant.repo")" \
+                    --secret id=contentsets,src="$(realpath src/yumrepos/content_sets.yaml)" \
+                    -v /etc/pki/ca-trust:/etc/pki/ca-trust:ro
+    fi
+
     if [ -n "$DIRECT" ]; then
         # turn on layer caching in the direct case; it wouldn't hurt in the
         # supermin path, but it'd be a waste of space on the rootfs
@@ -99,4 +147,4 @@ build_with_buildah() {
     rm -rf "${tempdir}"
 }
 
-build_with_buildah
+build_with_buildah

src/cmd-import

@@ -49,11 +49,16 @@ def main():
         # import into the tmp/repo to get the ostree-commit but also so it's cached
         ostree_commit = import_oci_archive(tmpd, tmp_oci_archive, buildid)
 
+        # artificially recreate generated lockfile
+        tmp_lockfile = generate_lockfile(tmpd, ostree_commit)
+
         # create meta.json
         build_meta = generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit)
 
         # move into official location
-        finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest)
+        finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest, tmp_lockfile)
+
+    subprocess.check_call(['/usr/lib/coreos-assembler/cmd-prune'])
 
 
 def parse_args():
@@ -83,6 +88,25 @@ def generate_oci_manifest(args, tmpd):
     return tmpf
 
 
+def generate_lockfile(tmpd, ostree_commit):
+    tmpf = os.path.join(tmpd, 'lockfile.json')
+
+    out = subprocess.check_output(['rpm-ostree', 'db', 'list', '--repo', 'tmp/repo', ostree_commit], encoding='utf-8')
+    rpmdb = {}
+    for line in out.splitlines():
+        if not line.startswith(' '):
+            continue
+        n, ev, ra = line.strip().rsplit('-', 2)
+        rpmdb[n] = {'evra': f'{ev}-{ra}'}
+
+    with open(tmpf, 'w') as f:
+        json.dump(fp=f, obj={
+            'packages': rpmdb
+        })
+
+    return tmpf
+
+
 def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit):
     name = metadata['Labels']['com.coreos.osname']
     buildid = metadata['Labels']['org.opencontainers.image.version']
@@ -115,7 +139,7 @@ def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_comm
     }
 
 
-def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest):
+def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest, tmp_lockfile):
     buildid = build_meta['buildid']
     arch = build_meta['coreos-assembler.basearch']
 
@@ -124,6 +148,7 @@ def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest):
 
     shutil.move(tmp_oci_archive, f'{destdir}/{build_meta['images']['ostree']['path']}')
     shutil.move(tmp_oci_manifest, f'{destdir}/{build_meta['images']['oci-manifest']['path']}')
+    shutil.move(tmp_lockfile, f'{destdir}/manifest-lock.generated.{arch}.json')
 
     with open(f'{destdir}/meta.json', 'w') as f:
         json.dump(build_meta, f, indent=4)

src/cmdlib.sh

@@ -1105,17 +1105,6 @@ cmdlib.import_ostree_commit(workdir, builddir, buildmeta, extract_json=('${extra
 ")
 }
 
-check_build_exists() {
-    local buildid=$1; shift
-    (python3 -c "
-import sys
-sys.path.insert(0, '${DIR}')
-from cosalib.builds import Builds
-builds = Builds('${workdir:-$(pwd)}')
-print(builds.has('${buildid}'))
-")
-}
-
 # Extract the value of NAME from os-release
 extract_osrelease_name() {
     local buildid=$1; shift