|
| 1 | +#!/usr/bin/python |
| 2 | + |
| 3 | +import argparse |
| 4 | +import json |
| 5 | +import os |
| 6 | +import sys |
| 7 | +import subprocess |
| 8 | +import yaml |
| 9 | + |
| 10 | +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| 11 | +from cosalib.cmdlib import get_basearch |
| 12 | + |
| 13 | + |
| 14 | +def format_packages_with_repoid(pkgs, repos): |
| 15 | + """ |
| 16 | + Takes a list of package URLs and repos and returns a list |
| 17 | + of package dictionaries with repoids. |
| 18 | + """ |
| 19 | + packages = [] |
| 20 | + local_repos = list(repos) |
| 21 | + if "fedora-coreos-pool" in local_repos: |
| 22 | + local_repos.remove("fedora-coreos-pool") |
| 23 | + |
| 24 | + if not local_repos: |
| 25 | + if pkgs: |
| 26 | + print("Error: No repos to associate with packages.", file=sys.stderr) |
| 27 | + sys.exit(1) |
| 28 | + return [] |
| 29 | + |
| 30 | + # We want to ensure that hermeto creates repo definitions for every repository. |
| 31 | + # A round-robin assignment ensures each repo is mentioned at least once. |
| 32 | + # This is needed because rpm-ostree uses the full list of repos to |
| 33 | + # resolve packages and errors out if a repository is missing. |
| 34 | + repo_numbers = len(local_repos) |
| 35 | + for i, pkg in enumerate(pkgs): |
| 36 | + packages.append({"url": pkg, "repoid": local_repos[i % repo_numbers]}) |
| 37 | + return packages |
| 38 | + |
| 39 | + |
| 40 | +def write_hermeto_lockfile(arch_packages, repos): |
| 41 | + """ |
| 42 | + Writes the hermeto lockfile structure. |
| 43 | + """ |
| 44 | + arches = [] |
| 45 | + for arch_data in arch_packages: |
| 46 | + arch = arch_data['arch'] |
| 47 | + pkgs = arch_data['packages'] |
| 48 | + formatted_packages = format_packages_with_repoid(pkgs, repos) |
| 49 | + arches.append({ |
| 50 | + 'arch': arch, |
| 51 | + 'packages': formatted_packages |
| 52 | + }) |
| 53 | + |
| 54 | + lockfile = { |
| 55 | + 'lockfileVersion': 1, |
| 56 | + "lockfileVendor": "redhat", |
| 57 | + "arches": arches |
| 58 | + } |
| 59 | + |
| 60 | + return lockfile |
| 61 | + |
| 62 | + |
| 63 | +def merge_lockfiles(base_lockfile, next_lockfile, override=False): |
| 64 | + """ |
| 65 | + Merges a lockfile into a base lockfile. |
| 66 | +
|
| 67 | + If is_override is True, it will only add packages to existing |
| 68 | + architectures. Otherwise, it will add new architectures. |
| 69 | + """ |
| 70 | + if not next_lockfile: |
| 71 | + return base_lockfile |
| 72 | + |
| 73 | + # Create a dictionary for base arches for easy lookup |
| 74 | + base_arches = {arch['arch']: arch for arch in base_lockfile.get('arches', [])} |
| 75 | + |
| 76 | + next_arches_list = next_lockfile.get('arches', []) |
| 77 | + if not next_arches_list: |
| 78 | + return base_lockfile |
| 79 | + |
| 80 | + for next_arch_entry in next_arches_list: |
| 81 | + if not isinstance(next_arch_entry, dict): |
| 82 | + continue |
| 83 | + arch = next_arch_entry.get('arch', None) |
| 84 | + if not arch: |
| 85 | + continue |
| 86 | + |
| 87 | + next_packages = next_arch_entry.get('packages', []) |
| 88 | + if arch in base_arches: |
| 89 | + # Arch exists, merge packages |
| 90 | + base_packages = base_arches[arch].get('packages', []) |
| 91 | + base_packages += next_packages |
| 92 | + elif not override: |
| 93 | + # Arch is new and this is not an override, so add it |
| 94 | + base_arches[arch] = next_arch_entry |
| 95 | + |
| 96 | + # Reconstruct the arches list |
| 97 | + base_lockfile['arches'] = list(base_arches.values()) |
| 98 | + return base_lockfile |
| 99 | + |
| 100 | + |
| 101 | +def query_packages_location(locks, repoquery_args): |
| 102 | + """ |
| 103 | + Resolves packages URLs for a given architecture. |
| 104 | + """ |
| 105 | + pkg_urls = [] |
| 106 | + if not locks: |
| 107 | + return pkg_urls |
| 108 | + |
| 109 | + locked_nevras = [f'{k}-{v.get('evra', '')}' for (k, v) in locks.items()] |
| 110 | + queryfmt = ["--queryformat", "%{name} %{location}\n"] |
| 111 | + cmd = ['dnf', 'repoquery'] + locked_nevras + repoquery_args + queryfmt |
| 112 | + result = subprocess.check_output(cmd, text=True) |
| 113 | + |
| 114 | + processed_urls = {} |
| 115 | + for line in result.split('\n'): |
| 116 | + # ignore empty lines |
| 117 | + if not line: |
| 118 | + continue |
| 119 | + name, url = line.split(' ') |
| 120 | + # Prioritize the url from fedora-coreos-pool |
| 121 | + # there is a bug in dnf here where the url returned is incorrect when the |
| 122 | + # repofile have more than one baseurl, which causes ppc64le and s390x |
| 123 | + # urls comming from fedora and fedora-updates to be invalid |
| 124 | + # See https://github.com/rpm-software-management/dnf5/issues/2466 |
| 125 | + existing_url = processed_urls.get(name, None) |
| 126 | + if 'coreos-pool' in url or not existing_url: |
| 127 | + processed_urls[name] = url |
| 128 | + |
| 129 | + pkg_urls = list(processed_urls.values()) |
| 130 | + # sanity check all the locked packages got resolved |
| 131 | + if len(pkg_urls) != len(locked_nevras): |
| 132 | + print("Some packages from the lockfile could not be resolved. The rpm-ostree lockfile is probably out of date.") |
| 133 | + sys.exit(1) |
| 134 | + |
| 135 | + print(f"Done. Resolved location for {len(pkg_urls)} packages.") |
| 136 | + return pkg_urls |
| 137 | + |
| 138 | + |
| 139 | +def get_locked_nevras(srcdir, arch): |
| 140 | + |
| 141 | + path = os.path.join(srcdir, f"manifest-lock.{arch}.json") |
| 142 | + |
| 143 | + data = {} |
| 144 | + try: |
| 145 | + with open(path, encoding='utf-8') as f: |
| 146 | + data = json.load(f) |
| 147 | + except IOError as e: |
| 148 | + print(f"Cannot read rpm-ostree lockfile. Reason: {e}") |
| 149 | + sys.exit(1) |
| 150 | + |
| 151 | + return data.get('packages', []) |
| 152 | + |
| 153 | + |
| 154 | +def generate_main(args): |
| 155 | + """ |
| 156 | + Generates the cachi2/hermeto RPM lock file. |
| 157 | + """ |
| 158 | + contextdir = args.context |
| 159 | + manifest = os.path.abspath(args.manifest) |
| 160 | + output_path = args.output |
| 161 | + arches = args.arch |
| 162 | + |
| 163 | + if not arches: |
| 164 | + arches_to_resolve = [get_basearch()] |
| 165 | + elif 'all' in arches: |
| 166 | + arches_to_resolve = ['x86_64', 'aarch64', 's390x', 'ppc64le'] |
| 167 | + else: |
| 168 | + arches_to_resolve = arches |
| 169 | + |
| 170 | + if os.path.exists(manifest): |
| 171 | + with open(manifest, 'r', encoding='utf-8') as f: |
| 172 | + manifest_data = json.load(f) |
| 173 | + else: |
| 174 | + print(f"flattened manifest not found at {manifest}") |
| 175 | + sys.exit(1) |
| 176 | + |
| 177 | + repos = manifest_data.get('repos', []) |
| 178 | + repos += manifest_data.get('lockfile-repos', []) |
| 179 | + |
| 180 | + # Tell dnf to load repos files from $contextdir |
| 181 | + repoquery_args = ["--refresh", "--quiet", f"--setopt=reposdir={contextdir}"] |
| 182 | + repoquery_args.extend([f"--repo={','.join(repos)}"]) |
| 183 | + |
| 184 | + packages = [] |
| 185 | + for arch in arches_to_resolve: |
| 186 | + locks = get_locked_nevras(contextdir, arch) |
| 187 | + if not locks: |
| 188 | + print(f"This tool derive the konflux lockfile from rpm-ostree lockfiles. Empty manifest-lock for {arch} in {contextdir}") |
| 189 | + sys.exit(1) |
| 190 | + print(f"Resolving packages for {arch}...") |
| 191 | + arch_args = [] |
| 192 | + if arch is not get_basearch(): |
| 193 | + # append noarch as well because otherwise those packages get excluded from results |
| 194 | + # We use --forcearch here because otherwise dnf still respect the system basearch |
| 195 | + # we have to specify both --arch and --forcearch to get both result for $arch and $noarch |
| 196 | + arch_args = ['--forcearch', arch, '--arch', arch, '--arch', 'noarch'] |
| 197 | + pkg_urls = query_packages_location(locks, repoquery_args + arch_args) |
| 198 | + packages.append({'arch': arch, 'packages': pkg_urls}) |
| 199 | + |
| 200 | + lockfile = write_hermeto_lockfile(packages, repos) |
| 201 | + |
| 202 | + try: |
| 203 | + with open(output_path, 'w', encoding='utf-8') as f: |
| 204 | + yaml.safe_dump(lockfile, f, default_flow_style=False) |
| 205 | + except IOError as e: |
| 206 | + print(f"\u274c Error: Could not write to output file '{output_path}'. Reason: {e}") |
| 207 | + sys.exit(1) |
| 208 | + |
| 209 | + |
| 210 | +def merge_main(args): |
| 211 | + """ |
| 212 | + Merges multiple lockfiles into one, optionally applying an override file. |
| 213 | + """ |
| 214 | + if not args.input: |
| 215 | + print("Error: at least one input file is required for merging.", file=sys.stderr) |
| 216 | + sys.exit(1) |
| 217 | + |
| 218 | + try: |
| 219 | + with open(args.input[0], 'r', encoding='utf-8') as f: |
| 220 | + base_lockfile = yaml.safe_load(f) |
| 221 | + except (IOError, yaml.YAMLError) as e: |
| 222 | + print(f"Error reading base lockfile {args.input[0]}: {e}", file=sys.stderr) |
| 223 | + sys.exit(1) |
| 224 | + |
| 225 | + for subsequent_file in args.input[1:]: |
| 226 | + try: |
| 227 | + with open(subsequent_file, 'r', encoding='utf-8') as f: |
| 228 | + next_lockfile = yaml.safe_load(f) |
| 229 | + base_lockfile = merge_lockfiles(base_lockfile, next_lockfile) |
| 230 | + except (IOError, yaml.YAMLError) as e: |
| 231 | + print(f"Error reading or merging {subsequent_file}: {e}", file=sys.stderr) |
| 232 | + sys.exit(1) |
| 233 | + |
| 234 | + if os.path.exists(args.override): |
| 235 | + try: |
| 236 | + with open(args.override, 'r', encoding="utf8") as f: |
| 237 | + override_data = yaml.safe_load(f) |
| 238 | + print(f"Merging override from {args.override}") |
| 239 | + base_lockfile = merge_lockfiles(base_lockfile, override_data, override=True) |
| 240 | + except (IOError, yaml.YAMLError) as e: |
| 241 | + print(f"Error reading or parsing override file '{args.override}': {e}", file=sys.stderr) |
| 242 | + sys.exit(1) |
| 243 | + |
| 244 | + try: |
| 245 | + with open(args.output, 'w', encoding='utf-8') as f: |
| 246 | + yaml.safe_dump(base_lockfile, f, default_flow_style=False) |
| 247 | + print(f"Successfully merged lockfiles to {args.output}") |
| 248 | + except IOError as e: |
| 249 | + print(f"Error writing to output file '{args.output}': {e}", file=sys.stderr) |
| 250 | + sys.exit(1) |
| 251 | + |
| 252 | + |
| 253 | +if __name__ == "__main__": |
| 254 | + parser = argparse.ArgumentParser( |
| 255 | + description="Generate and merge hermeto lock files." |
| 256 | + ) |
| 257 | + subparsers = parser.add_subparsers(dest='command', required=True) |
| 258 | + |
| 259 | + # GENERATE command |
| 260 | + parser_generate = subparsers.add_parser( |
| 261 | + 'generate', |
| 262 | + help='Resolve RPMs and generate a lockfile for one or more architectures.' |
| 263 | + ) |
| 264 | + parser_generate.add_argument( |
| 265 | + 'manifest', |
| 266 | + help='Path to the flattened rpm-ostree manifest (e.g., tmp/manifest.json)' |
| 267 | + ) |
| 268 | + parser_generate.add_argument( |
| 269 | + '--context', |
| 270 | + default='.', |
| 271 | + help="Path to the directory containing repofiles and lockfiles. (default: '.')" |
| 272 | + ) |
| 273 | + parser_generate.add_argument( |
| 274 | + '--output', |
| 275 | + default='./rpms.lock.yaml', |
| 276 | + help="Path for the hermeto lockfile. (default: './rpms.lock.yaml')" |
| 277 | + ) |
| 278 | + parser_generate.add_argument( |
| 279 | + '--arch', |
| 280 | + action='append', |
| 281 | + choices=['x86_64', 'aarch64', 's390x', 'ppc64le', 'all'], |
| 282 | + help="The architecture to resolve. Can be specified multiple times. 'all' resolves all architectures." |
| 283 | + ) |
| 284 | + parser_generate.set_defaults(func=generate_main) |
| 285 | + |
| 286 | + # MERGE command |
| 287 | + parser_merge = subparsers.add_parser( |
| 288 | + 'merge', |
| 289 | + help='Merge multiple architecture-specific lockfiles into a single file.' |
| 290 | + ) |
| 291 | + parser_merge.add_argument( |
| 292 | + '--input', |
| 293 | + nargs='+', |
| 294 | + required=True, |
| 295 | + help='One or more input lockfiles to merge.' |
| 296 | + ) |
| 297 | + parser_merge.add_argument( |
| 298 | + '--output', |
| 299 | + default='./rpms.lock.yaml', |
| 300 | + help="Path for the merged lockfile. (default: './rpms.lock.yaml')" |
| 301 | + ) |
| 302 | + parser_merge.add_argument( |
| 303 | + '--override', |
| 304 | + default='konflux-lockfile-override.yaml', |
| 305 | + help="Path to an override file. (default: 'konflux-lockfile-override.yaml')" |
| 306 | + ) |
| 307 | + parser_merge.set_defaults(func=merge_main) |
| 308 | + |
| 309 | + args = parser.parse_args() |
| 310 | + args.func(args) |
0 commit comments