From c51cd19dbcbbd849e00702f3adf8c45e52d1cd2a Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 11 Aug 2025 20:02:35 -0400 Subject: [PATCH 01/13] cmd-diff: refactor diff_cmd_output into a loop It had some common elements so let's use a loop. --- src/cmd-diff | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index 29d39f835d..d1ce6dbaf1 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -404,15 +404,13 @@ def diff_metal(diff_from, diff_to): def diff_cmd_outputs(cmd, file_from, file_to): with tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as f_from, \ tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as f_to: - if '{}' not in cmd: - cmd += ['{}'] - idx = cmd.index('{}') - cmd_from = list(cmd) - cmd_from[idx] = file_from - subprocess.run(cmd_from, check=True, stdout=f_from).stdout - cmd_to = list(cmd) - cmd_to[idx] = file_to - subprocess.run(cmd_to, check=True, stdout=f_to).stdout + for file, output in (file_from, f_from), (file_to, f_to): + c = list(cmd) + if '{}' not in c: + c += ['{}'] + idx = c.index('{}') + c[idx] = file + subprocess.run(c, check=True, stdout=file).stdout git_diff(f_from.name, f_to.name) From 539ed6abaed4d1f4a5369faa7402ca401c8c6083 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 11 Aug 2025 20:18:11 -0400 Subject: [PATCH 02/13] cmd-diff: rename vars in diff_cmd_outputs Since we could be operating on directories or files change file_from -> path_from and file_to -> path_to. Also change the temporary output filenames to more properly indicate they are outputs. --- src/cmd-diff | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index d1ce6dbaf1..b84e4ffdc5 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -401,17 +401,17 @@ def diff_metal(diff_from, diff_to): shutdown_process(p_to) -def diff_cmd_outputs(cmd, file_from, file_to): - with tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as f_from, \ - tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as f_to: - for file, output in (file_from, f_from), (file_to, f_to): +def diff_cmd_outputs(cmd, path_from, path_to): + with tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as from_output, \ + tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as to_output: + for path, output in (path_from, from_output), (path_to, to_output): c = list(cmd) if '{}' not in c: c += ['{}'] idx = c.index('{}') - c[idx] = file - subprocess.run(c, check=True, stdout=file).stdout - git_diff(f_from.name, f_to.name) + c[idx] = path + subprocess.run(c, check=True, stdout=output).stdout + git_diff(from_output.name, to_output.name) def git_diff(arg_from, arg_to): From 01196598007847b7367cebae9e0a4a18ef0d749b Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 11 Aug 2025 20:24:32 -0400 Subject: [PATCH 03/13] cmd-diff: support `cd` strategy for diff_cmd_outputs This strategy simply changes directory into the given path before running the provided command rather than replacing a templated `{}` with the path. Useful for commands that operate more cleanly when operated on in the directory where you want the operation to occur. --- src/cmd-diff | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index b84e4ffdc5..2e98a4b4db 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -37,6 +37,11 @@ class OSTreeImport(IntEnum): FULL = 3 +class DiffCmdOutputStrategy(IntEnum): + CD = 1 + TEMPLATE = 2 + + @dataclass class Differ: name: str @@ -401,16 +406,20 @@ def diff_metal(diff_from, diff_to): shutdown_process(p_to) -def diff_cmd_outputs(cmd, path_from, path_to): +def diff_cmd_outputs(cmd, path_from, path_to, strategy: DiffCmdOutputStrategy = DiffCmdOutputStrategy.TEMPLATE): + workingdir = None with tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as from_output, \ tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as to_output: for path, output in (path_from, from_output), (path_to, to_output): c = list(cmd) - if '{}' not in c: - c += ['{}'] - idx = c.index('{}') - c[idx] = path - subprocess.run(c, check=True, stdout=output).stdout + if strategy == DiffCmdOutputStrategy.TEMPLATE: + if '{}' not in c: + c += ['{}'] + idx = c.index('{}') + c[idx] = path + elif strategy == DiffCmdOutputStrategy.CD: + workingdir = path + subprocess.run(c, cwd=workingdir, check=True, stdout=output).stdout git_diff(from_output.name, to_output.name) From 0a2cf58e7ebb4d9cd94840b544fc18c13cfb238f Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 11 Aug 2025 23:00:17 -0400 Subject: [PATCH 04/13] cmd-diff: support `git difftool` so we can utilize vimdiff Having vimdiff is a lot more powerful to me because you can manually do a few things in the terminal to massage the files on each side of the diff to give you more information (i.e. narrow in on exactly what diff you are interested in). Let's add a --difftool boolean flag to trigger `git difftool`, which will allow diffs to be displayed using vimdiff. Additionally include vim-enhanced so we have `vimdiff` installed and at our disposal. --- src/cmd-diff | 15 ++++++++++++++- src/deps.txt | 3 +++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/cmd-diff b/src/cmd-diff index 2e98a4b4db..4a81e8c9fe 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -54,11 +54,20 @@ TMP_REPO = 'tmp/repo' DIFF_CACHE = 'tmp/diff-cache' +# Global variable that can be set once and direct the underlying code to leverage +# a difftool with git diffing or not. This could be an argument passed around to +# underlying functions, but decided to just implement it as a global variable for now. +USE_DIFFTOOL = False + def main(): args = parse_args() builds = Builds() + # Modify the USE_DIFFTOOL global based on the --difftool argument + global USE_DIFFTOOL + USE_DIFFTOOL = args.difftool + latest_build = builds.get_latest() os.makedirs(DIFF_CACHE, exist_ok=True) @@ -114,6 +123,7 @@ def parse_args(): parser.add_argument("--to", dest='diff_to', help="Second build ID") parser.add_argument("--gc", action='store_true', help="Delete cached diff content") parser.add_argument("--arch", dest='arch', help="Architecture of builds") + parser.add_argument("--difftool", action='store_true', help="Use git difftool") for differ in DIFFERS: parser.add_argument("--" + differ.name, action='store_true', default=False, @@ -424,7 +434,10 @@ def diff_cmd_outputs(cmd, path_from, path_to, strategy: DiffCmdOutputStrategy = def git_diff(arg_from, arg_to): - runcmd(['git', 'diff', '--no-index', arg_from, arg_to], check=False) + subcmd = 'diff' + if USE_DIFFTOOL: + subcmd = 'difftool' + runcmd(['git', subcmd, '--no-index', arg_from, arg_to], check=False) def cache_dir(dir): diff --git a/src/deps.txt b/src/deps.txt index 3f9400c7e5..81aa1b0c7b 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -111,3 +111,6 @@ python3-libguestfs # For generating kubernetes YAML files (e.g Konflux resources) kustomize + +# For vimdiff +vim-enhanced From e81f65db8a6ad91ef58a304f35533fb5ad1525dd Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 11 Aug 2025 23:27:09 -0400 Subject: [PATCH 05/13] cmd-diff: refactor metal mounting code into helper This way we can have more commands that can leverae this code for different "diffs" on the resulting mounted filesystems. Prep for a future commit. --- src/cmd-diff | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index 4a81e8c9fe..ff66a3ee39 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -364,7 +364,10 @@ def run_guestfs_mount(image_path, mount_target): g.close() -def diff_metal(diff_from, diff_to): +# Generator that will mount up metal image filesystems and yield +# the paths to be used for analysis and then clean up once given back +# control. +def diff_metal_helper(diff_from, diff_to): metal_from = get_metal_path(diff_from) metal_to = get_metal_path(diff_to) @@ -397,8 +400,8 @@ def diff_metal(diff_from, diff_to): if not p.is_alive(): raise Exception(f"A guestfs process for {os.path.basename(d)} died unexpectedly.") - # Now that the mounts are live, we can diff them - git_diff(mount_dir_from, mount_dir_to) + # Allow the caller to operate on these values + yield mount_dir_from, mount_dir_to finally: # Unmount the FUSE binds, this will make the guestfs mount calls return @@ -416,6 +419,11 @@ def diff_metal(diff_from, diff_to): shutdown_process(p_to) +def diff_metal(diff_from, diff_to): + for mount_dir_from, mount_dir_to in diff_metal_helper(diff_from, diff_to): + git_diff(mount_dir_from, mount_dir_to) + + def diff_cmd_outputs(cmd, path_from, path_to, strategy: DiffCmdOutputStrategy = DiffCmdOutputStrategy.TEMPLATE): workingdir = None with tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as from_output, \ From 80cb2832ac8df11fef4e302161fe47a9b89e0180 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 12 Aug 2025 00:07:24 -0400 Subject: [PATCH 06/13] cmd-diff: support --metal-du and --metal-ls A few more views of differences between two metal disk image mounted filesystems. --- src/cmd-diff | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/cmd-diff b/src/cmd-diff index ff66a3ee39..ae94502d8c 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -424,6 +424,18 @@ def diff_metal(diff_from, diff_to): git_diff(mount_dir_from, mount_dir_to) +def diff_metal_du(diff_from, diff_to): + for mount_dir_from, mount_dir_to in diff_metal_helper(diff_from, diff_to): + cmd = ['find', '.', '-type', 'd', '-exec', 'du', '-sh', '{}', ';'] + diff_cmd_outputs(cmd, mount_dir_from, mount_dir_to, strategy=DiffCmdOutputStrategy.CD) + + +def diff_metal_ls(diff_from, diff_to): + for mount_dir_from, mount_dir_to in diff_metal_helper(diff_from, diff_to): + cmd = ['find', '.'] + diff_cmd_outputs(cmd, mount_dir_from, mount_dir_to, strategy=DiffCmdOutputStrategy.CD) + + def diff_cmd_outputs(cmd, path_from, path_to, strategy: DiffCmdOutputStrategy = DiffCmdOutputStrategy.TEMPLATE): workingdir = None with tempfile.NamedTemporaryFile(prefix=cmd[0] + '-') as from_output, \ @@ -485,6 +497,10 @@ DIFFERS = [ needs_ostree=OSTreeImport.NO, function=diff_metal_partitions), Differ("metal", "Diff metal disk image content", needs_ostree=OSTreeImport.NO, function=diff_metal), + Differ("metal-du", "Compare directory usage of metal disk image content", + needs_ostree=OSTreeImport.NO, function=diff_metal_du), + Differ("metal-ls", "Compare directory listing of metal disk image content", + needs_ostree=OSTreeImport.NO, function=diff_metal_ls), ] if __name__ == '__main__': From 3042eb17f53ceb415c2fc08775a37a6780775b1a Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 12 Aug 2025 10:33:50 -0400 Subject: [PATCH 07/13] cmd-diff: fix superfluous .stdout call The .stdout at the end of this line is unnecessary and has no effect. When stdout is redirected to a file object, the stdout attribute of the returned CompletedProcess object is None. Assisted-By https://github.com/coreos/coreos-assembler/pull/4253#discussion_r2268528878 --- src/cmd-diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd-diff b/src/cmd-diff index ae94502d8c..4ef2d43378 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -449,7 +449,7 @@ def diff_cmd_outputs(cmd, path_from, path_to, strategy: DiffCmdOutputStrategy = c[idx] = path elif strategy == DiffCmdOutputStrategy.CD: workingdir = path - subprocess.run(c, cwd=workingdir, check=True, stdout=output).stdout + subprocess.run(c, cwd=workingdir, check=True, stdout=output) git_diff(from_output.name, to_output.name) From b8687f9adec259912f8b979e7c392a4f29be249b Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 18 Sep 2025 16:46:08 -0400 Subject: [PATCH 08/13] cmd-diff: default --from to N-1 build This way if it's not specified some sane value is used. --- src/cmd-diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd-diff b/src/cmd-diff index 4ef2d43378..5ae4177396 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -78,7 +78,7 @@ def main(): args.diff_from = builds.get_previous() args.diff_to = latest_build elif args.diff_from is None: - args.diff_from = latest_build + args.diff_from = builds.get_previous() elif args.diff_to is None: args.diff_to = latest_build From d2fddd4ed3debed331177afa9bc95c8f389f7f25 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 18 Sep 2025 16:47:17 -0400 Subject: [PATCH 09/13] cmd-diff: support --rpms-json for outputting JSON This basically maps to passing `--format=json` to `rpm-ostree db diff`. Useful for updating meta.json with info about package differences and advisories. --- src/cmd-diff | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index 5ae4177396..7978b4b42e 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -158,10 +158,21 @@ def diff_source_control(diff_from, diff_to): print(f" --> {origin_url}/compare/{config_from['commit'][:7]}...{config_to['commit'][:7]}") -def diff_rpms(diff_from, diff_to): +def diff_rpms_json(diff_from, diff_to): + diff_rpms(diff_from, diff_to, json=True) + + +def diff_rpms_no_json(diff_from, diff_to): + diff_rpms(diff_from, diff_to, json=False) + + +def diff_rpms(diff_from, diff_to, json): ref_from = diff_from.id ref_to = diff_to.id - runcmd(['rpm-ostree', 'db', 'diff', '--repo', TMP_REPO, ref_from, ref_to]) + cmd = ['rpm-ostree', 'db', 'diff', '--repo', TMP_REPO, ref_from, ref_to] + if json: + cmd.append('--format=json') + runcmd(cmd) def diff_ostree_ls(diff_from, diff_to): @@ -468,7 +479,10 @@ def cache_dir(dir): # unfortunately, this has to come at the end to resolve functions DIFFERS = [ - Differ("rpms", "Diff RPMs", needs_ostree=OSTreeImport.FULL, function=diff_rpms), + Differ("rpms", "Diff RPMs", + needs_ostree=OSTreeImport.FULL, function=diff_rpms_no_json), + Differ("rpms-json", "Diff RPMs, output JSON", + needs_ostree=OSTreeImport.FULL, function=diff_rpms_json), Differ("source-control", "Diff config and COSA input commits", needs_ostree=OSTreeImport.NO, function=diff_source_control), Differ("ostree-ls", "Diff OSTree contents using 'ostree diff'", From 86d8d31c557b99ecd1dd30985f89f838d9829026 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 25 Sep 2025 16:45:04 -0400 Subject: [PATCH 10/13] cosalib/builds: add get_build_commitmeta() Similar to get_build_meta(), let's make a convenience function for grabbing commitmeta. --- src/cosalib/builds.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cosalib/builds.py b/src/cosalib/builds.py index f87776257c..2fd1cbdab1 100644 --- a/src/cosalib/builds.py +++ b/src/cosalib/builds.py @@ -94,6 +94,11 @@ def get_build_meta(self, build_id, basearch=None): with open(os.path.join(d, 'meta.json')) as f: return json.load(f) + def get_build_commitmeta(self, build_id, basearch=None): + d = self.get_build_dir(build_id, basearch) + with open(os.path.join(d, 'commitmeta.json')) as f: + return json.load(f) + def get_tags(self): return self._data.get('tags', []) From a8441e6ff38fd4de72fd873826143f0ec97bf7f2 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 25 Sep 2025 17:06:33 -0400 Subject: [PATCH 11/13] cmd-diff: make rpm diff function names more specific This set of function names leverages rpm-ostree for the diffing. --- src/cmd-diff | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index 7978b4b42e..2eceedd5ab 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -158,15 +158,15 @@ def diff_source_control(diff_from, diff_to): print(f" --> {origin_url}/compare/{config_from['commit'][:7]}...{config_to['commit'][:7]}") -def diff_rpms_json(diff_from, diff_to): - diff_rpms(diff_from, diff_to, json=True) +def diff_rpms_rpm_ostree_json(diff_from, diff_to): + diff_rpms_rpm_ostree(diff_from, diff_to, json=True) -def diff_rpms_no_json(diff_from, diff_to): - diff_rpms(diff_from, diff_to, json=False) +def diff_rpms_rpm_ostree_no_json(diff_from, diff_to): + diff_rpms_rpm_ostree(diff_from, diff_to, json=False) -def diff_rpms(diff_from, diff_to, json): +def diff_rpms_rpm_ostree(diff_from, diff_to, json): ref_from = diff_from.id ref_to = diff_to.id cmd = ['rpm-ostree', 'db', 'diff', '--repo', TMP_REPO, ref_from, ref_to] @@ -480,9 +480,9 @@ def cache_dir(dir): # unfortunately, this has to come at the end to resolve functions DIFFERS = [ Differ("rpms", "Diff RPMs", - needs_ostree=OSTreeImport.FULL, function=diff_rpms_no_json), + needs_ostree=OSTreeImport.FULL, function=diff_rpms_rpm_ostree_no_json), Differ("rpms-json", "Diff RPMs, output JSON", - needs_ostree=OSTreeImport.FULL, function=diff_rpms_json), + needs_ostree=OSTreeImport.FULL, function=diff_rpms_rpm_ostree_json), Differ("source-control", "Diff config and COSA input commits", needs_ostree=OSTreeImport.NO, function=diff_source_control), Differ("ostree-ls", "Diff OSTree contents using 'ostree diff'", From 0adf9fe750bf8297f8fabf6afdfb631adaf3052e Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 25 Sep 2025 13:20:27 -0400 Subject: [PATCH 12/13] cmd-diff: Add new ability to generate rpm diffs from commitmeta.json This adds two new arguments for generating rpm-ostree db diff compatible output by just leveraging the information in the commitmeta.json files in each build's directory. This is a much more lightweight way to get diffs rather than having to download and import the entire ociarchive, which we are now having to do in the build-with-buildah world where the rpm information isn't available to us in the OSTree commit object like it was in the past. This was assisted by Google/Gemini and got me 75% of the way there. The prompt provided: ``` Look at the files old-commitmeta.json and new-commitmeta.json and generate a two functions in src/cmd-diff that will produce similar diff output. The first function will produce human readable diff output and should be in the form as shown in the human-diff.txt file. The second function should produce JSON output and be in the format as shown in the json-diff.txt file. The commitmeta.json files will exist in the build directory for a build (so when diffing two builds you'll need to look at the commitmeta.json for the first build and then the commitmeta.json for the second build and then produce the diff output based on the differences in those two files. ``` Assisted-By: --- src/cmd-diff | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/src/cmd-diff b/src/cmd-diff index 2eceedd5ab..348b73a027 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -3,10 +3,12 @@ import argparse import os import shutil +import json import subprocess import sys import tempfile import time +import rpm from multiprocessing import Process from dataclasses import dataclass @@ -24,11 +26,78 @@ class DiffBuildTarget: id: str dir: str meta: dict + commitmeta: dict @staticmethod def from_build(builds, build, arch): return DiffBuildTarget(build, builds.get_build_dir(build, arch), - builds.get_build_meta(build, arch)) + builds.get_build_meta(build, arch), + builds.get_build_commitmeta(build, arch)) + + +class PackageDiffType(IntEnum): + ADD = 0 + REMOVE = 1 + UPGRADE = 2 + DOWNGRADE = 3 + + +@dataclass +class Package: + name: str + epoch: int + v: str # Version + r: str # Release + a: str # Architecture + + def evr(self): + epoch_string = "" if self.epoch == "0" else f"{self.epoch}:" + return f"{epoch_string}{self.v}-{self.r}" + + # Compare versions + # rc > 0 -> Newer; rc == 0 -> Same; rc < 0 -> Older + @staticmethod + def vercmp(from_pkg, to_pkg): + # pylint: disable=E1101 + return rpm.labelCompare((to_pkg.epoch, to_pkg.v, to_pkg.r), + (from_pkg.epoch, from_pkg.v, from_pkg.r)) + + # Return a data sctructure representing a diff entry from the + # rpm-ostree db diff --json output. + # "pkgdiff" : [ + # [ + # "NetworkManager", + # 2, + # { + # "PreviousPackage" : [ + # "NetworkManager", + # "1:1.52.0-5.el9_6", + # "x86_64" + # ], + # "NewPackage" : [ + # "NetworkManager", + # "1:1.52.0-7.el9_6", + # "x86_64" + # ] + # } + # ], + @staticmethod + def to_json_diff_entry(difftype: PackageDiffType, from_pkg, to_pkg): + name = from_pkg.name if from_pkg else to_pkg.name + change = "" + match difftype: + case PackageDiffType.ADD: + change = {"NewPackage": [to_pkg.name, to_pkg.evr(), to_pkg.a]} + case PackageDiffType.REMOVE: + change = {"PreviousPackage": [from_pkg.name, from_pkg.evr(), from_pkg.a]} + case PackageDiffType.UPGRADE | PackageDiffType.DOWNGRADE: + change = { + "PreviousPackage": [from_pkg.name, from_pkg.evr(), from_pkg.a], + "NewPackage": [to_pkg.name, to_pkg.evr(), to_pkg.a] + } + case _: + raise Exception(f"Invalid PackageDiffType: {difftype}") + return [name, difftype, change] class OSTreeImport(IntEnum): @@ -175,6 +244,89 @@ def diff_rpms_rpm_ostree(diff_from, diff_to, json): runcmd(cmd) +def diff_rpms_commitmeta_json(diff_from, diff_to): + diff_rpms_commitmeta(diff_from, diff_to, json_output=True) + + +def diff_rpms_commitmeta_no_json(diff_from, diff_to): + diff_rpms_commitmeta(diff_from, diff_to, json_output=False) + + +def diff_rpms_commitmeta(diff_from, diff_to, json_output=False): + """ + Diff the commitmeta.json files between two builds. + """ + from_meta = diff_from.meta + to_meta = diff_to.meta + from_commitmeta = diff_from.commitmeta + to_commitmeta = diff_to.commitmeta + + # Gather info about upgraded, downgraded, added, removed RPMS + from_rpms = {pkg[0]: Package(*pkg) for pkg in from_commitmeta.get('rpmostree.rpmdb.pkglist', [])} + to_rpms = {pkg[0]: Package(*pkg) for pkg in to_commitmeta.get('rpmostree.rpmdb.pkglist', [])} + + added_rpms = set(to_rpms.keys()) - set(from_rpms.keys()) + removed_rpms = set(from_rpms.keys()) - set(to_rpms.keys()) + common_rpms = set(from_rpms.keys()) & set(to_rpms.keys()) + + upgraded_rpms = [] + downgraded_rpms = [] + for name in common_rpms: + from_pkg = from_rpms[name] + to_pkg = to_rpms[name] + rc = Package.vercmp(from_pkg, to_pkg) + if rc > 0: + upgraded_rpms.append((name, from_pkg, to_pkg)) + elif rc == 0: + pass # They are equal versions + elif rc < 0: + downgraded_rpms.append((name, from_pkg, to_pkg)) + + if json_output: + diff = { + "ostree-commit-from": from_meta.get('ostree-commit'), + "ostree-commit-to": to_meta.get('ostree-commit'), + "pkgdiff": [], + "advisories": [] + } + for name in sorted(added_rpms): + diff['pkgdiff'].append(Package.to_json_diff_entry(PackageDiffType.ADD, from_pkg=None, to_pkg=to_rpms[name])) + for name in sorted(removed_rpms): + diff['pkgdiff'].append(Package.to_json_diff_entry(PackageDiffType.REMOVE, from_pkg=from_rpms[name], to_pkg=None)) + for name, from_pkg, to_pkg in sorted(upgraded_rpms): + diff['pkgdiff'].append(Package.to_json_diff_entry(PackageDiffType.UPGRADE, from_pkg=from_pkg, to_pkg=to_pkg)) + for name, from_pkg, to_pkg in sorted(downgraded_rpms): + diff['pkgdiff'].append(Package.to_json_diff_entry(PackageDiffType.DOWNGRADE, from_pkg=from_pkg, to_pkg=to_pkg)) + + # Only add advisory info for the JSON output + from_advisories = {adv[0]: adv for adv in from_commitmeta.get('rpmostree.advisories', [])} + to_advisories = {adv[0]: adv for adv in to_commitmeta.get('rpmostree.advisories', [])} + added_advisories = set(to_advisories.keys()) - set(from_advisories.keys()) + for name in added_advisories: + diff['advisories'].append(to_advisories[name]) + # Dump the JSON to stdout + print(json.dumps(diff, indent=2)) + else: + print(f"ostree diff commit from: {diff_from.id} ({from_meta.get('ostree-commit')})") + print(f"ostree diff commit to: {diff_to.id} ({to_meta.get('ostree-commit')})") + if upgraded_rpms: + print("Upgraded:") + for name, from_pkg, to_pkg in sorted(upgraded_rpms): + print(f" {name} {from_pkg.evr()} -> {to_pkg.evr()}") + if downgraded_rpms: + print("Downgraded:") + for name, from_pkg, to_pkg in sorted(downgraded_rpms): + print(f" {name} {from_pkg.evr()} -> {to_pkg.evr()}") + if removed_rpms: + print("Removed:") + for name in sorted(removed_rpms): + print(f" {name}-{from_rpms[name].evr()}") + if added_rpms: + print("Added:") + for name in sorted(added_rpms): + print(f" {name}-{to_rpms[name].evr()}") + + def diff_ostree_ls(diff_from, diff_to): commit_from = diff_from.meta['ostree-commit'] commit_to = diff_to.meta['ostree-commit'] @@ -483,6 +635,10 @@ DIFFERS = [ needs_ostree=OSTreeImport.FULL, function=diff_rpms_rpm_ostree_no_json), Differ("rpms-json", "Diff RPMs, output JSON", needs_ostree=OSTreeImport.FULL, function=diff_rpms_rpm_ostree_json), + Differ("rpms-commitmeta", "Diff rpms from commitmeta.json", + needs_ostree=OSTreeImport.NO, function=diff_rpms_commitmeta_no_json), + Differ("rpms-json-commitmeta", "Diff RPMS & Advisories from commitmeta.json, output JSON", + needs_ostree=OSTreeImport.NO, function=diff_rpms_commitmeta_json), Differ("source-control", "Diff config and COSA input commits", needs_ostree=OSTreeImport.NO, function=diff_source_control), Differ("ostree-ls", "Diff OSTree contents using 'ostree diff'", From 5ba9aacb292e17f1b43d79019a8d41f01ddd46f0 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 25 Sep 2025 17:12:28 -0400 Subject: [PATCH 13/13] cmd-diff: promote commitmeta.json version of rpm diff Since this version is more lightweight (doesn't require OSTree import) let's make is the primary one that gets called when someone does `cosa diff --rpms`. --- src/cmd-diff | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cmd-diff b/src/cmd-diff index 348b73a027..c259c7f184 100755 --- a/src/cmd-diff +++ b/src/cmd-diff @@ -631,13 +631,13 @@ def cache_dir(dir): # unfortunately, this has to come at the end to resolve functions DIFFERS = [ - Differ("rpms", "Diff RPMs", + Differ("rpms-rpm-ostree", "Diff RPMs using rpm-ostree", needs_ostree=OSTreeImport.FULL, function=diff_rpms_rpm_ostree_no_json), - Differ("rpms-json", "Diff RPMs, output JSON", + Differ("rpms-rpm-ostree-json", "Diff RPMs & Advisories using rpm-ostree, output JSON", needs_ostree=OSTreeImport.FULL, function=diff_rpms_rpm_ostree_json), - Differ("rpms-commitmeta", "Diff rpms from commitmeta.json", + Differ("rpms", "Diff rpms from commitmeta.json", needs_ostree=OSTreeImport.NO, function=diff_rpms_commitmeta_no_json), - Differ("rpms-json-commitmeta", "Diff RPMS & Advisories from commitmeta.json, output JSON", + Differ("rpms-json", "Diff RPMS & Advisories from commitmeta.json, output JSON", needs_ostree=OSTreeImport.NO, function=diff_rpms_commitmeta_json), Differ("source-control", "Diff config and COSA input commits", needs_ostree=OSTreeImport.NO, function=diff_source_control),