upload-oscontainer: Support oscontainer.yaml and extensions.yaml

Pairs with openshift/os#455

Add an `oscontainer.yaml` which allows configuring (currently) just
the `FROM` line equivalent.  This helps keep things declarative
in the config instead of part of the pipeline.

And since extensions are now a core part of OpenShift 4 (particularly RT kernel)
which also uses the oscontainer, let's lift the logic to generate
those bits of the oscontainer into this repo and out of the config git.

Both of these are part of the general philosophy we've had that:

 - config git is declarative config
 - coreos-assembler is the mechanism
 - a pipeline just scripts coreos-assembler in a way that's
   easy to reproduce with plain podman too

Author: cgwalters

src/cmd-upload-oscontainer

@@ -7,6 +7,7 @@
 
 import argparse
 import json
+import yaml
 import os
 import shutil
 import subprocess
@@ -39,6 +40,24 @@ metapath = f"{latest_build_path}/meta.json"
 with open(metapath) as f:
     meta = json.load(f)
 
+configdir = os.path.abspath('src/config')
+oscconfigpath = f'{configdir}/oscontainer.yaml'
+if os.path.exists(oscconfigpath):
+    with open(oscconfigpath) as f:
+        c = yaml.safe_load(f)
+        base = c.get('base')
+        if base is not None:
+            args.from_image = base
+
+extensions_src = 'src/config/extensions.yaml'
+extensions_destdir = None
+if os.path.exists(extensions_src):
+    extensions_destdir = 'tmp/extensions'
+    if os.path.exists(extensions_destdir):
+        shutil.rmtree(extensions_destdir)
+    os.mkdir(extensions_destdir)
+    cmdlib.run_verbose(['/usr/lib/coreos-assembler/download-extensions', extensions_destdir])
+
 print("Preparing to upload oscontainer for build: {}".format(latest_build))
 ostree_commit = meta['ostree-commit']
 
@@ -87,6 +106,8 @@ os.environ['REGISTRY_AUTH_FILE'] = authfile
 cosa_argv.extend(['/usr/lib/coreos-assembler/oscontainer.py', '--workdir=./tmp', 'build',  f"--from={args.from_image}"])
 for d in args.add_directory:
     cosa_argv.append(f"--add-directory={d}")
+if extensions_destdir is not None:
+    cosa_argv.append(f"--add-directory={extensions_destdir}")
 cosa_argv.append(f"--display-name={display_name}")
 subprocess.check_call(cosa_argv +
     [f'--digestfile={digestfile}',

src/download-extensions

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# RPMs as operating system extensions, distinct from the base ostree commit/image
+# https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md
+
+import os
+import sys
+import yaml
+from cosalib import cmdlib
+
+destdir = sys.argv[1]
+tmpdir = 'tmp'
+# yum wants this to be absolute
+configdir = os.path.abspath('src/config')
+extsrcpath = f'{configdir}/extensions.yaml'
+extjson = f'{tmpdir}/extensions.json'
+basearch = cmdlib.get_basearch()
+
+with open(extsrcpath) as f:
+    extensions = yaml.safe_load(f)
+
+# The "v2" format here is that there's an extensions/ directory, with subdirectories
+# for each extension - except you should ignore "repodata/".
+edestdir = f'{destdir}/extensions'
+os.mkdir(edestdir)
+
+# Stuff that's not part of the extension
+dependenciesdir = f'{edestdir}/dependencies'
+os.mkdir(dependenciesdir)
+
+
+# Downloads packages from specified repos
+def yumdownload(destdir, pkgs):
+    # FIXME eventually use rpm-ostree for this
+    # shellcheck disable=SC2068
+    args = ['yum', f'--setopt=reposdir={configdir}', f'--arch={basearch}', 'download']
+    args.extend(pkgs)
+    cmdlib.run_verbose(args, cwd=destdir)
+
+
+# Reuseable function for setting up an extension
+# Assumes it is running in "${destdir}/extensions"
+# 1 = extension name
+# 2 = package string/glob
+# 3 = OPTIONAL: dependencies string/glob
+def createext(extname, pkgs):
+    print(f"Creating extension {extname}")
+    extdir = f"{edestdir}/{extname}"
+    os.mkdir(extdir)
+    primary = pkgs[0]
+    yumdownload(extdir, [primary])
+
+    deps = pkgs[1:]
+    if len(deps) > 0:
+        print(f"Downloading dependencies for {extname}")
+        yumdownload(dependenciesdir, deps)
+
+
+for (name, ext) in extensions['extensions'].items():
+    pkgs = ext['packages']
+    extarches = ext.get('architectures')
+    if extarches is not None and basearch not in extarches:
+        print(f"Skipping extension {name} for this architecture")
+        continue
+    createext(name, pkgs)
+
+# Create the yum/dnf repo
+cmdlib.run_verbose(['createrepo_c', '--no-database', '.'], cwd=destdir)