Add cosa import

This command takes as argument a `containers-transport(5)`-style pullspec
and creates a new cosa build dir from it. It essentially bridges the gap
between coreos/fedora-coreos-config#3348 and the
rest of the cosa pipeline.

Co-authored-by: Jonathan Lebon <[email protected]>

Author: PeaceRebel

cmd/coreos-assembler.go

@@ -13,7 +13,7 @@ import (
 
 // commands we'd expect to use in the local dev path
 var buildCommands = []string{"init", "fetch", "build", "osbuild", "run", "prune", "clean", "list"}
-var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container"}
+var advancedBuildCommands = []string{"import", "buildfetch", "buildupload", "oc-adm-release", "push-container"}
 var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "oraclecloud", "qemu", "secex", "virtualbox", "vmware", "vultr"}
 
 var utilityCommands = []string{"aws-replicate", "coreos-prune", "compress", "copy-container", "diff", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-session", "sign", "tag", "update-variant"}

pkg/builds/cosa_v1.go

@@ -1,7 +1,7 @@
 package builds
 
 // generated by 'make schema'
-// source hash: 4bb5641ee8c32b122c412cbaefe3d068685ea6cbffcf3162e600a01268c92146
+// source hash: 445150ada0fe019c7bb33c793185b312111ed7538a59e1a0b424c10c6c2dbc0d
 
 type AdvisoryDiff []AdvisoryDiffItems
 
@@ -56,6 +56,7 @@ type Build struct {
 	CosaDelayedMetaMerge      bool                  `json:"coreos-assembler.delayed-meta-merge,omitempty"`
 	CosaImageChecksum         string                `json:"coreos-assembler.image-config-checksum,omitempty"`
 	CosaImageVersion          int                   `json:"coreos-assembler.image-genver,omitempty"`
+	CosaImportedOciImage      bool                  `json:"coreos-assembler.oci-imported,omitempty"`
 	Extensions                *Extensions           `json:"extensions,omitempty"`
 	ExtensionsContainer       *PrimaryImage         `json:"extensions-container,omitempty"`
 	FedoraCoreOsParentCommit  string                `json:"fedora-coreos.parent-commit,omitempty"`
@@ -64,15 +65,15 @@ type Build struct {
 	GitDirty                  string                `json:"coreos-assembler.config-dirty,omitempty"`
 	IbmCloud                  []Cloudartifact       `json:"ibmcloud,omitempty"`
 	ImageInputChecksum        string                `json:"coreos-assembler.image-input-checksum,omitempty"`
-	InputHashOfTheRpmOstree   string                `json:"rpm-ostree-inputhash"`
+	InputHashOfTheRpmOstree   string                `json:"rpm-ostree-inputhash,omitempty"`
 	Koji                      *Koji                 `json:"koji,omitempty"`
 	KubevirtContainer         *PrimaryImage         `json:"kubevirt,omitempty"`
 	MetaStamp                 float64               `json:"coreos-assembler.meta-stamp,omitempty"`
 	Name                      string                `json:"name"`
 	Oscontainer               *PrimaryImage         `json:"oscontainer,omitempty"`
 	OstreeCommit              string                `json:"ostree-commit"`
 	OstreeContentBytesWritten int                   `json:"ostree-content-bytes-written,omitempty"`
-	OstreeContentChecksum     string                `json:"ostree-content-checksum"`
+	OstreeContentChecksum     string                `json:"ostree-content-checksum,omitempty"`
 	OstreeNCacheHits          int                   `json:"ostree-n-cache-hits,omitempty"`
 	OstreeNContentTotal       int                   `json:"ostree-n-content-total,omitempty"`
 	OstreeNContentWritten     int                   `json:"ostree-n-content-written,omitempty"`

pkg/builds/schema_doc.go

@@ -1,5 +1,5 @@
 // Generated by ./generate-schema.sh
-// Source hash: 4bb5641ee8c32b122c412cbaefe3d068685ea6cbffcf3162e600a01268c92146
+// Source hash: 445150ada0fe019c7bb33c793185b312111ed7538a59e1a0b424c10c6c2dbc0d
 // DO NOT EDIT
 
 package builds
@@ -255,10 +255,8 @@ var generatedSchemaJSON = `{
     "buildid",
     "name",
     "ostree-commit",
-    "ostree-content-checksum",
     "ostree-timestamp",
-    "ostree-version",
-    "rpm-ostree-inputhash"
+    "ostree-version"
   ],
   "optional": [
     "aliyun",
@@ -273,13 +271,15 @@ var generatedSchemaJSON = `{
     "images",
     "koji",
     "oscontainer",
+    "ostree-content-checksum",
     "extensions",
     "extensions-container",
     "parent-pkgdiff",
     "pkgdiff",
     "parent-advisories-diff",
     "advisories-diff",
     "release-payload",
+    "rpm-ostree-inputhash",
     "summary",
     "s3",
     "coreos-assembler.basearch",
@@ -297,6 +297,7 @@ var generatedSchemaJSON = `{
     "coreos-assembler.meta-stamp",
     "coreos-assembler.overrides-active",
     "coreos-assembler.yumrepos-git",
+    "coreos-assembeler.oci-imported",
     "fedora-coreos.parent-commit",
     "fedora-coreos.parent-version",
     "ref"
@@ -378,6 +379,12 @@ var generatedSchemaJSON = `{
       "default": "",
       "minLength": 1
     },
+    "coreos-assembler.oci-imported": {
+      "$id": "#/properties/coreos-assembler.oci-imported",
+      "type": "boolean",
+      "title": "COSA imported OCI image",
+      "default": "False"
+    },
     "coreos-assembler.code-source": {
       "$id": "#/properties/coreos-assembler.code-source",
       "type": "string",

src/cmd-coreos-prune

@@ -55,7 +55,7 @@ Build = collections.namedtuple("Build", ["id", "images", "arch", "meta_json"])
 # set metadata caching to 5m
 CACHE_MAX_AGE_METADATA = 60 * 5
 # These lists are up to date as of schema hash
-# 4bb5641ee8c32b122c412cbaefe3d068685ea6cbffcf3162e600a01268c92146. If changing
+# 445150ada0fe019c7bb33c793185b312111ed7538a59e1a0b424c10c6c2dbc0d. If changing
 # this hash, ensure that the list of SUPPORTED and UNSUPPORTED artifacts below
 # is up to date.
 SUPPORTED = ["amis", "aws-winli", "gcp"]

src/cmd-import

@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+
+'''
+This command takes a containers-transports(5) ref to an OCI image and converts
+it into a `cosa build`, as if one did `cosa build ostree`. One can then e.g.
+`cosa buildextend-qemu` right away.
+'''
+
+import argparse
+import datetime
+import json
+import os
+import subprocess
+import tempfile
+import shutil
+import sys
+from stat import (
+    S_IREAD,
+    S_IRGRP,
+    S_IROTH)
+from cosalib.builds import Builds
+from cosalib.cmdlib import (
+    rfc3339_time,
+    get_basearch,
+    sha256sum_file,
+    import_oci_archive)
+
+
+def main():
+    args = parse_args()
+
+    # immediate inspect to error out early if it doesn't exists/we don't have ACLs and to do some upfront checks
+    metadata = skopeo_inspect(args.srcimg)
+
+    # let raise if missing
+    assert metadata['Labels']['containers.bootc'] == '1'
+    buildid = metadata['Labels']['org.opencontainers.image.version']
+
+    builds = Builds()
+    if builds.has(buildid):
+        print(f"ERROR: Build ID {buildid} already exists!")
+        sys.exit(1)
+
+    with tempfile.TemporaryDirectory(prefix='cosa-import-', dir='tmp') as tmpd:
+        # create the OCI archive and manifest
+        tmp_oci_archive = generate_oci_archive(args, tmpd)
+        tmp_oci_manifest = generate_oci_manifest(args, tmpd)
+
+        # import into the tmp/repo to get the ostree-commit but also so it's cached
+        ostree_commit = import_oci_archive(tmpd, tmp_oci_archive, buildid)
+
+        # create meta.json
+        build_meta = generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit)
+
+        # move into official location
+        finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(prog='cosa import')
+    parser.add_argument("srcimg", metavar='IMAGE',
+                        help="image to import (containers-transports(5) format)")
+    return parser.parse_args()
+
+
+def generate_oci_archive(args, tmpd):
+    tmpf = os.path.join(tmpd, 'out.ociarchive')
+    subprocess.check_call(['skopeo', 'copy', '--preserve-digests', args.srcimg,
+                           f"oci-archive:{tmpf}"])
+    return tmpf
+
+
+def generate_oci_manifest(args, tmpd):
+    tmpf = os.path.join(tmpd, 'oci-manifest.json')
+    with open(tmpf, 'wb') as f:
+        f.write(subprocess.check_output(["skopeo", "inspect", "--raw", args.srcimg]))
+        os.fchmod(f.fileno(), S_IREAD | S_IRGRP | S_IROTH)
+    return tmpf
+
+
+def generate_build_meta(tmp_oci_archive, tmp_oci_manifest, metadata, ostree_commit):
+    name = metadata['Labels']['com.coreos.osname']
+    buildid = metadata['Labels']['org.opencontainers.image.version']
+    created_timestamp = parse_timestamp(metadata['Created'])
+    arch = get_basearch()
+
+    return {
+        'ostree-commit': ostree_commit,
+        'ostree-version': buildid,
+        'buildid': buildid,
+        'name': name,
+        'coreos-assembler.basearch': arch,
+        'coreos-assembler.build-timestamp': created_timestamp,
+        'coreos-assembler.oci-imported': True,
+        'ostree-timestamp': created_timestamp,
+        'images': {
+            'ostree': {
+                "path": f"{name}-{buildid}-ostree.{arch}.ociarchive",
+                "sha256": sha256sum_file(tmp_oci_archive),
+                'size': os.path.getsize(tmp_oci_archive),
+                "skip-compression": True
+            },
+            'oci-manifest': {
+                'path': f'{name}-{buildid}-ostree.{arch}-manifest.json',
+                'sha256': sha256sum_file(tmp_oci_manifest),
+                'size': os.path.getsize(tmp_oci_manifest),
+                "skip-compression": True,
+            },
+        },
+    }
+
+
+def finalize_build(builds, build_meta, tmp_oci_archive, tmp_oci_manifest):
+    buildid = build_meta['buildid']
+    arch = build_meta['coreos-assembler.basearch']
+
+    destdir = f'builds/{buildid}/{arch}'
+    os.makedirs(destdir)
+
+    shutil.move(tmp_oci_archive, f'{destdir}/{build_meta['images']['ostree']['path']}')
+    shutil.move(tmp_oci_manifest, f'{destdir}/{build_meta['images']['oci-manifest']['path']}')
+
+    with open(f'{destdir}/meta.json', 'w') as f:
+        json.dump(build_meta, f, indent=4)
+
+    # and finally the real deal: insert the build and bump latest symlink
+    builds.insert_build(buildid, arch)
+    builds.bump_timestamp()
+
+    if os.path.exists('builds/latest'):
+        os.remove('builds/latest')
+    os.symlink(f'{buildid}', 'builds/latest', target_is_directory=True)
+
+    print(f'Imported OCI image as build {buildid}')
+
+
+def skopeo_inspect(image):
+    return json.loads(subprocess.check_output(['skopeo', 'inspect', '-n', image]))
+
+
+def parse_timestamp(timestamp):
+    # datetime's doesn't support nanoseconds.
+    # So trim it.
+    if len(timestamp) > 26 and timestamp[19] == '.':
+        timestamp = timestamp[:26] + "Z"
+
+    timestamp = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
+    return rfc3339_time(timestamp.replace(tzinfo=datetime.timezone.utc))
+
+
+if __name__ == '__main__':
+    main()

src/cosalib/cmdlib.py

@@ -295,12 +295,14 @@ def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=True, parti
               lifetime=LOCK_DEFAULT_LIFETIME):
         repo = os.path.join(tmpdir, 'repo')
         commit = buildmeta['ostree-commit']
+        is_oci_imported = buildmeta.get('coreos-assembler.oci-imported', False)
         tarfile = os.path.join(buildpath, buildmeta['images']['ostree']['path'])
         # create repo in case e.g. tmp/ was cleared out; idempotent
         subprocess.check_call(['ostree', 'init', '--repo', repo, '--mode=archive'])
 
-        # in the common case where we're operating on a recent build, the OSTree
-        # commit should already be in the tmprepo
+        # in the common case where we're operating on a recent build (or
+        # recently imported OCI image), the OSTree commit should already be in
+        # the tmprepo
         commitpartial = os.path.join(repo, f'state/{commit}.commitpartial')
         if (subprocess.call(['ostree', 'show', '--repo', repo, commit],
                             stdout=subprocess.DEVNULL,
@@ -332,8 +334,12 @@ def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=True, parti
         # We do this in two stages, because right now ex-container only writes to
         # non-archive repos.  Also, in the privileged case we need sudo to write
         # to `repo-build`, though it might be good to change this by default.
-        if os.environ.get('COSA_PRIVILEGED', '') == '1':
+        if is_oci_imported:
+            import_oci_archive(tmpdir, tarfile, buildmeta['buildid'])
+        elif os.environ.get('COSA_PRIVILEGED', '') == '1':
             build_repo = os.path.join(repo, '../../cache/repo-build')
+            # note: this actually is the same as `container unencapsulate` and
+            # so only works with "pure OSTree OCI" encapsulated commits (legacy path)
             subprocess.check_call(['sudo', 'ostree', 'container', 'import', '--repo', build_repo,
                                    '--write-ref', buildmeta['buildid'],
                                    'ostree-unverified-image:oci-archive:' + tarfile])
@@ -354,6 +360,42 @@ def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=True, parti
             extract_image_json(workdir, commit)
 
 
+def import_oci_archive(parent_tmpd, ociarchive, ref):
+    with tempfile.TemporaryDirectory(dir=parent_tmpd) as tmpd:
+        subprocess.check_call(['ostree', 'init', '--repo', tmpd, '--mode=bare-user'])
+
+        # Init tmp/repo in case it doesn't exist.
+        # If it exists, no problem. It's idempotent
+        subprocess.check_call(['ostree', 'init', '--repo', 'tmp/repo', '--mode=archive'])
+
+        # import all the blob refs for more efficient import into bare-user repo
+        blob_refs = subprocess.check_output(['ostree', 'refs', '--repo', 'tmp/repo',
+                                             '--list', 'ostree/container/blob'],
+                                            encoding='utf-8').splitlines()
+        if len(blob_refs) > 0:
+            subprocess.check_call(['ostree', 'pull-local', '--repo', tmpd, 'tmp/repo'] + blob_refs)
+
+        subprocess.check_call(['ostree', 'container', 'image', 'pull', tmpd,
+                               f'ostree-unverified-image:oci-archive:{ociarchive}'])
+
+        # awkwardly work around the fact that there is no --write-ref equivalent
+        refs = subprocess.check_output(['ostree', 'refs', '--repo', tmpd,
+                                        '--list', 'ostree/container/image'],
+                                       encoding='utf-8').splitlines()
+        assert len(refs) == 1
+        subprocess.check_call(['ostree', 'refs', '--repo', tmpd, refs[0], '--create', ref])
+        subprocess.check_call(['ostree', 'refs', '--repo', 'tmp/repo', ref, '--delete'])
+        subprocess.check_call(['ostree', 'pull-local', '--repo', 'tmp/repo', tmpd, ref])
+
+        # export back all the blob refs for more efficient imports of next builds
+        blob_refs = subprocess.check_output(['ostree', 'refs', '--repo', tmpd,
+                                             '--list', 'ostree/container/blob'],
+                                            encoding='utf-8').splitlines()
+        subprocess.check_call(['ostree', 'pull-local', '--repo', 'tmp/repo', tmpd] + blob_refs)
+
+    return subprocess.check_output(['ostree', 'rev-parse', '--repo', 'tmp/repo', ref], encoding='utf-8').strip()
+
+
 def get_basearch():
     try:
         return get_basearch.saved

src/v1.json

@@ -249,10 +249,8 @@
     "buildid",
     "name",
     "ostree-commit",
-    "ostree-content-checksum",
     "ostree-timestamp",
-    "ostree-version",
-    "rpm-ostree-inputhash"
+    "ostree-version"
   ],
   "optional": [
     "aliyun",
@@ -267,13 +265,15 @@
     "images",
     "koji",
     "oscontainer",
+    "ostree-content-checksum",
     "extensions",
     "extensions-container",
     "parent-pkgdiff",
     "pkgdiff",
     "parent-advisories-diff",
     "advisories-diff",
     "release-payload",
+    "rpm-ostree-inputhash",
     "summary",
     "s3",
     "coreos-assembler.basearch",
@@ -291,6 +291,7 @@
     "coreos-assembler.meta-stamp",
     "coreos-assembler.overrides-active",
     "coreos-assembler.yumrepos-git",
+    "coreos-assembeler.oci-imported",
     "fedora-coreos.parent-commit",
     "fedora-coreos.parent-version",
     "ref"
@@ -372,6 +373,12 @@
       "default": "",
       "minLength": 1
     },
+    "coreos-assembler.oci-imported": {
+      "$id": "#/properties/coreos-assembler.oci-imported",
+      "type": "boolean",
+      "title": "COSA imported OCI image",
+      "default": "False"
+    },
     "coreos-assembler.code-source": {
       "$id": "#/properties/coreos-assembler.code-source",
       "type": "string",