cmd-fetch: Derive konflux lockfiles from rpm-ostree

src/cmd-fetch

@@ -39,7 +39,8 @@ Usage: coreos-assembler fetch --help
   --write-lockfile-to=FILE Write updated base lockfile to separate file
   --with-cosa-overrides    Don't ignore cosa overrides in `overrides/rpm`
   --autolock=VERSION       If no base lockfile used, create one from any arch build of `VERSION`
-
+  --konflux                Generate hermeto lockfile for Konflux derived from the rpm-ostree lockfiles.
+                           Auto enabled if `rpms.lock.yaml` is found in the config directory.
 EOF
 }
 
@@ -50,8 +51,9 @@ IGNORE_COSA_OVERRIDES_ARG=--ignore-cosa-overrides
 DRY_RUN=
 FORCE_ARG=
 STRICT=
+KONFLUX=
 rc=0
-options=$(getopt --options h --longoptions help,update-lockfile,dry-run,with-cosa-overrides,write-lockfile-to:,strict,force,autolock: -- "$@") || rc=$?
+options=$(getopt --options h --longoptions help,update-lockfile,dry-run,with-cosa-overrides,write-lockfile-to:,strict,force,autolock:,konflux -- "$@") || rc=$?
 [ $rc -eq 0 ] || {
     print_help
     exit 1
@@ -87,6 +89,9 @@ while true; do
             shift;
             AUTOLOCK_VERSION=$1
             ;;
+        --konflux)
+            KONFLUX=1
+            ;;
         --)
             shift
             break
@@ -105,6 +110,7 @@ fi
 
 prepare_build
 
+
 lock_args=
 extra_args=
 
@@ -176,3 +182,11 @@ if [ -n "${UPDATE_LOCKFILE}" ]; then
     (cd "${workdir}" && mv -f "${tmprepo}/tmp/manifest-lock.json" "${outfile}")
     echo "Wrote out lockfile ${outfile}"
 fi
+
+KONFLUX_LOCKFILE=rpms.lock.yaml
+if [ -n "${KONFLUX}" ] || [ -f "${configdir}/${KONFLUX_LOCKFILE}" ]; then
+   echo "Generating hermeto lockfile..."
+   /usr/lib/coreos-assembler/konflux-rpm-lockfile generate "${flattened_manifest}" --context "${configdir}" --output "${tmprepo}/tmp/${arch}.${KONFLUX_LOCKFILE}"
+   mv -f "${tmprepo}/tmp/${arch}.${KONFLUX_LOCKFILE}" "${configdir}/${arch}.${KONFLUX_LOCKFILE}"
+   echo "Wrote out hermeto (konflux) lockfile: ${configdir}/${arch}.${KONFLUX_LOCKFILE}"
+fi

src/cosalib/cmdlib.py

@@ -575,3 +575,29 @@ def ensure_glob(pathname, **kwargs):
 def ncpu():
     '''Return the number of usable CPUs we have for parallelism.'''
     return int(subprocess.check_output(['kola', 'ncpu']))
+
+
+def get_locked_nevras(srcdir, arch=None):
+    """
+    Gathers all locked packages from the manifest-lock files.
+    The return format can be a dictionary of {pkgname: evr}
+    """
+    if not arch:
+        arch = get_basearch()
+    lockfile_path = os.path.join(srcdir, f"manifest-lock.{arch}.json")
+    overrides_path = os.path.join(srcdir, "manifest-lock.overrides.yaml")
+    overrides_arch_path = os.path.join(srcdir, f"manifest-lock.overrides.{arch}.json")
+
+    locks = {}
+    for path in [lockfile_path, overrides_path, overrides_arch_path]:
+        if os.path.exists(path):
+            with open(path, encoding='utf-8') as f:
+                if path.endswith('.yaml'):
+                    data = yaml.safe_load(f)
+                else:
+                    data = json.load(f)
+                # this essentially re-implements the merge semantics of rpm-ostree
+                locks.update({pkgname: v.get('evra') or v.get('evr')
+                              for (pkgname, v) in data['packages'].items()})
+
+    return locks

src/konflux-rpm-lockfile

@@ -0,0 +1,293 @@
+#!/usr/bin/python
+
+import argparse
+import json
+import os
+import sys
+import subprocess
+import yaml
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from cosalib.cmdlib import get_locked_nevras, get_basearch
+
+
+def format_packages_with_repoid(pkgs, repos):
+    """
+    Takes a list of package URLs and repos and returns a list
+    of package dictionaries with repoids.
+    """
+    packages = []
+    local_repos = list(repos)
+    if "fedora-coreos-pool" in local_repos:
+        local_repos.remove("fedora-coreos-pool")
+
+    if not local_repos:
+        if pkgs:
+            print("Error: No repos to associate with packages.", file=sys.stderr)
+            sys.exit(1)
+        return []
+
+    # We want to ensure that hermeto creates repo definitions for every repository.
+    # A round-robin assignment ensures each repo is mentioned at least once.
+    # This is needed because rpm-ostree uses the full list of repos to
+    # resolve packages and errors out if a repository is missing.
+    repo_numbers = len(local_repos)
+    for i, pkg in enumerate(pkgs):
+        packages.append({"url": pkg, "repoid": local_repos[i % repo_numbers]})
+    return packages
+
+
+def write_hermeto_lockfile(arch_packages, repos):
+    """
+    Writes the hermeto lockfile structure.
+    """
+    arches = []
+    for arch_data in arch_packages:
+        arch = arch_data['arch']
+        pkgs = arch_data['packages']
+        formatted_packages = format_packages_with_repoid(pkgs, repos)
+        arches.append({
+            'arch': arch,
+            'packages': formatted_packages
+        })
+
+    lockfile = {
+        'lockfileVersion': 1,
+        "lockfileVendor": "redhat",
+        "arches": arches
+    }
+
+    return lockfile
+
+
+def merge_lockfiles(base_lockfile, next_lockfile, override=False):
+    """
+    Merges a lockfile into a base lockfile.
+
+    If is_override is True, it will only add packages to existing
+    architectures. Otherwise, it will add new architectures.
+    """
+    if not next_lockfile:
+        return base_lockfile
+
+    # Create a dictionary for base arches for easy lookup
+    base_arches = {arch['arch']: arch for arch in base_lockfile.get('arches', [])}
+
+    next_arches_list = next_lockfile.get('arches', [])
+    if not next_arches_list:
+        return base_lockfile
+
+    for next_arch_entry in next_arches_list:
+        if not isinstance(next_arch_entry, dict):
+            continue
+        arch = next_arch_entry.get('arch', None)
+        if not arch:
+            continue
+
+        next_packages = next_arch_entry.get('packages', [])
+        if arch in base_arches:
+            # Arch exists, merge packages
+            base_packages = base_arches[arch].get('packages', [])
+            base_packages += next_packages
+        elif not override:
+            # Arch is new and this is not an override, so add it
+            base_arches[arch] = next_arch_entry
+
+    # Reconstruct the arches list
+    base_lockfile['arches'] = list(base_arches.values())
+    return base_lockfile
+
+
+def query_packages_location(locks, repoquery_args):
+    """
+    Resolves packages URLs for a given architecture.
+    """
+    pkg_urls = []
+    if not locks:
+        return pkg_urls
+
+    locked_nevras = [f'{k}-{v}' for (k, v) in locks.items()]
+    cmd = ['dnf', 'repoquery'] + locked_nevras + repoquery_args
+    result = subprocess.check_output(cmd, text=True)
+
+    processed_urls = {}
+    for line in result.strip().split('\n'):
+        # ignore empty lines
+        if not line:
+            continue
+        parts = line.split(' ', 1)
+        # If more than 1 URL is returned we pick the first one
+        if len(parts) == 2:
+            name, url = parts
+            if name not in processed_urls:
+                processed_urls[name] = url
+    pkg_urls = list(processed_urls.values())
+    # sanity check all the locked packages got resolved
+    if len(pkg_urls) != len(locked_nevras):
+        print("Some packages from the lockfile could not be resolved. The rpm-ostree lockfile is probably out of date.")
+        sys.exit(1)
+
+    print(f"Done. Resolved location for {len(pkg_urls)} packages.")
+    return pkg_urls
+
+
+def generate_main(args):
+    """
+    Generates the cachi2/hermeto RPM lock file.
+    """
+    contextdir = args.context
+    manifest = os.path.abspath(args.manifest)
+    output_path = args.output
+    arches = args.arch
+
+    if not arches:
+        arches_to_resolve = [get_basearch()]
+    elif 'all' in arches:
+        arches_to_resolve = ['x86_64', 'aarch64', 's390x', 'ppc64le']
+    else:
+        arches_to_resolve = arches
+
+    if os.path.exists(manifest):
+        with open(manifest, 'r', encoding='utf-8') as f:
+            manifest_data = json.load(f)
+    else:
+        print(f"flattened manifest not found at {manifest}")
+        sys.exit(1)
+
+    repos = manifest_data.get('repos', [])
+    repos += manifest_data.get('lockfile-repos', [])
+
+    repoquery_args = ["--queryformat", "%{name} %{location}\n", "--disablerepo=*", "--refresh", "--quiet"]
+    # Tell dnf to load repos files from $contextdir
+    repoquery_args.extend([f"--setopt=reposdir={contextdir}"])
+
+    for repoid in set(repos):
+        repoquery_args.extend([f"--enablerepo={repoid}"])
+
+    packages = []
+    for arch in arches_to_resolve:
+        locks = get_locked_nevras(contextdir, arch)
+        if not locks:
+            print(f"This tool derive the konflux lockfile from rpm-ostree lockfiles. No manifest-lock exist for {arch} in {contextdir}")
+            sys.exit(1)
+        print(f"Resolving packages for {arch}...")
+        arch_args = []
+        if arch is not get_basearch():
+            # append noarch as well because otherwise those packages get excluded from results
+            # We use --forcearch here because otherwise dnf still respect the system basearch
+            # we have to specify both --arch and --forcearch to get both result for $arch and $noarch
+            arch_args = ['--forcearch', arch, '--arch', arch, '--arch', 'noarch']
+        pkg_urls = query_packages_location(locks, repoquery_args + arch_args)
+        packages.append({'arch': arch, 'packages': pkg_urls})
+
+    lockfile = write_hermeto_lockfile(packages, repos)
+
+    try:
+        with open(output_path, 'w', encoding='utf-8') as f:
+            yaml.safe_dump(lockfile, f, default_flow_style=False)
+    except IOError as e:
+        print(f"\u274c Error: Could not write to output file '{output_path}'. Reason: {e}")
+        sys.exit(1)
+
+
+def merge_main(args):
+    """
+    Merges multiple lockfiles into one, optionally applying an override file.
+    """
+    if not args.input:
+        print("Error: at least one input file is required for merging.", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        with open(args.input[0], 'r', encoding='utf-8') as f:
+            base_lockfile = yaml.safe_load(f)
+    except (IOError, yaml.YAMLError) as e:
+        print(f"Error reading base lockfile {args.input[0]}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    for subsequent_file in args.input[1:]:
+        try:
+            with open(subsequent_file, 'r', encoding='utf-8') as f:
+                next_lockfile = yaml.safe_load(f)
+            base_lockfile = merge_lockfiles(base_lockfile, next_lockfile)
+        except (IOError, yaml.YAMLError) as e:
+            print(f"Error reading or merging {subsequent_file}: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    if os.path.exists(args.override):
+        try:
+            with open(args.override, 'r', encoding="utf8") as f:
+                override_data = yaml.safe_load(f)
+            print(f"Merging override from {args.override}")
+            base_lockfile = merge_lockfiles(base_lockfile, override_data, override=True)
+        except (IOError, yaml.YAMLError) as e:
+            print(f"Error reading or parsing override file '{args.override}': {e}", file=sys.stderr)
+            sys.exit(1)
+
+    try:
+        with open(args.output, 'w', encoding='utf-8') as f:
+            yaml.safe_dump(base_lockfile, f, default_flow_style=False)
+        print(f"Successfully merged lockfiles to {args.output}")
+    except IOError as e:
+        print(f"Error writing to output file '{args.output}': {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate and merge hermeto lock files."
+    )
+    subparsers = parser.add_subparsers(dest='command', required=True)
+
+    # GENERATE command
+    parser_generate = subparsers.add_parser(
+        'generate',
+        help='Resolve RPMs and generate a lockfile for one or more architectures.'
+    )
+    parser_generate.add_argument(
+        'manifest',
+        help='Path to the flattened rpm-ostree manifest (e.g., tmp/manifest.json)'
+    )
+    parser_generate.add_argument(
+        '--context',
+        default='.',
+        help="Path to the directory containing repofiles and lockfiles. (default: '.')"
+    )
+    parser_generate.add_argument(
+        '--output',
+        default='./rpms.lock.yaml',
+        help="Path for the hermeto lockfile. (default: './rpms.lock.yaml')"
+    )
+    parser_generate.add_argument(
+        '--arch',
+        action='append',
+        choices=['x86_64', 'aarch64', 's390x', 'ppc64le', 'all'],
+        help="The architecture to resolve. Can be specified multiple times. 'all' resolves all architectures."
+    )
+    parser_generate.set_defaults(func=generate_main)
+
+    # MERGE command
+    parser_merge = subparsers.add_parser(
+        'merge',
+        help='Merge multiple architecture-specific lockfiles into a single file.'
+    )
+    parser_merge.add_argument(
+        '--input',
+        nargs='+',
+        required=True,
+        help='One or more input lockfiles to merge.'
+    )
+    parser_merge.add_argument(
+        '--output',
+        default='./rpms.lock.yaml',
+        help="Path for the merged lockfile. (default: './rpms.lock.yaml')"
+    )
+    parser_merge.add_argument(
+        '--override',
+        default='konflux-lockfile-override.yaml',
+        help="Path to an override file. (default: 'konflux-lockfile-override.yaml')"
+    )
+    parser_merge.set_defaults(func=merge_main)
+
+    args = parser.parse_args()
+    args.func(args)