ENT-13632: Added generate release information

src/cfengine_cli/dev.py

@@ -1,5 +1,7 @@
 import os
-from cfbs.commands import generate_release_information_command
+from cfengine_cli.masterfiles.generate_release_information import (
+    generate_release_information_impl,
+)
 from cfengine_cli.utils import UserError
 from cfengine_cli.deptool import (
     update_dependency_tables as _update_dependency_tables,
@@ -8,6 +10,13 @@
 from cfengine_cli.docs import update_docs, check_docs
 
 
+def generate_release_information_command(
+    omit_download=False, check=False, min_version=None
+):
+    generate_release_information_impl(omit_download, check, min_version)
+    return 0
+
+
 def _continue_prompt() -> bool:
     answer = None
     while answer not in ("y", "n", "yes", "no"):

src/cfengine_cli/masterfiles/analyze.py

@@ -0,0 +1,93 @@
+from collections import OrderedDict
+import os
+
+from cfbs.utils import dict_sorted_by_key, file_sha256, version_as_comparable_list
+
+Version = str
+
+
+def initialize_vcf():
+    versions_dict = {"versions": {}}
+    checksums_dict = {"checksums": {}}
+    files_dict = {"files": {}}
+
+    return versions_dict, checksums_dict, files_dict
+
+
+def versions_checksums_files(
+    files_dir_path, version, versions_dict, checksums_dict, files_dict
+):
+    for root, _, files in os.walk(files_dir_path):
+        for name in files:
+            full_relpath = os.path.join(root, name)
+            tarball_relpath = os.path.relpath(full_relpath, files_dir_path)
+            file_checksum = file_sha256(full_relpath)
+
+            if version not in versions_dict["versions"]:
+                versions_dict["versions"][version] = {}
+            versions_dict["versions"][version][tarball_relpath] = file_checksum
+
+            if file_checksum not in checksums_dict["checksums"]:
+                checksums_dict["checksums"][file_checksum] = {}
+            if tarball_relpath not in checksums_dict["checksums"][file_checksum]:
+                checksums_dict["checksums"][file_checksum][tarball_relpath] = []
+            checksums_dict["checksums"][file_checksum][tarball_relpath].append(version)
+
+            if tarball_relpath not in files_dict["files"]:
+                files_dict["files"][tarball_relpath] = {}
+            if file_checksum not in files_dict["files"][tarball_relpath]:
+                files_dict["files"][tarball_relpath][file_checksum] = []
+            files_dict["files"][tarball_relpath][file_checksum].append(version)
+
+    return versions_dict, checksums_dict, files_dict
+
+
+def finalize_vcf(versions_dict, checksums_dict, files_dict):
+    # explicitly sort VCF data to ensure determinism
+
+    # checksums.json:
+    working_dict = checksums_dict["checksums"]
+    for c in working_dict.keys():
+        for f in working_dict[c].keys():
+            # sort each version list, descending
+            working_dict[c][f] = sorted(
+                working_dict[c][f],
+                key=lambda v: version_as_comparable_list(v),
+                reverse=True,
+            )
+        # sort filepaths, alphabetically
+        working_dict[c] = dict_sorted_by_key(working_dict[c])
+    # sort checksums
+    checksums_dict["checksums"] = dict_sorted_by_key(working_dict)
+
+    # files.json:
+    working_dict = files_dict["files"]
+    # sort each list, first by version descending, then by checksum
+    for f in working_dict.keys():
+        for c in working_dict[f].keys():
+            # sort each version list, descending
+            working_dict[f][c] = sorted(
+                working_dict[f][c],
+                key=lambda v: version_as_comparable_list(v),
+                reverse=True,
+            )
+        # sort checksums
+        working_dict[f] = dict_sorted_by_key(working_dict[f])
+    # sort files, alphabetically
+    files_dict["files"] = dict_sorted_by_key(working_dict)
+
+    # versions.json:
+    working_dict = versions_dict["versions"]
+    # sort files of each version
+    for v in working_dict.keys():
+        working_dict[v] = dict_sorted_by_key(working_dict[v])
+    # sort version numbers, in decreasing order
+    versions_dict["versions"] = OrderedDict(
+        sorted(
+            working_dict.items(),
+            key=lambda p: version_as_comparable_list(p[0]),
+            reverse=True,
+        )
+    )
+
+    return versions_dict, checksums_dict, files_dict

src/cfengine_cli/masterfiles/check_download_matches_git.py

@@ -0,0 +1,108 @@
+import os
+from collections import OrderedDict
+
+from cfbs.utils import (
+    dict_diff,
+    read_json,
+    CFBSExitError,
+    write_json,
+    version_as_comparable_list,
+)
+
+
+def check_download_matches_git(versions):
+    """Check that the downloadable files match the git files.
+
+    This can be used to monitor / detect if something has been changed, accidentally or maliciously.
+
+    Generates a `differences-*.txt` file for each version.
+    """
+    assert os.path.isfile("versions.json")
+    assert os.path.isfile("versions-git.json")
+
+    download_versions_dict = read_json("versions.json")
+    git_versions_dict = read_json("versions-git.json")
+
+    assert download_versions_dict is not None
+    assert git_versions_dict is not None
+
+    diffs_dict = {"differences": {}}
+
+    nonmatching_versions = []
+    extraneous_count = 0
+    differing_count = 0
+
+    for version in versions:
+        dl_version_files_dict = download_versions_dict["versions"][version]
+        git_version_files_dict = git_versions_dict["versions"][version]
+
+        # normalize downloaded version dictionary filepaths
+        # necessary because the downloaded version and git version dictionaries have filepaths of different forms
+        new_download_dict = {}
+        for key, value in dl_version_files_dict.items():
+            if key.startswith("masterfiles/"):
+                key = key[12:]
+            new_download_dict[key] = value
+        dl_version_files_dict = new_download_dict
+
+        version_diffs_dict = {}
+        version_diffs_dict["files_only_in_downloads"] = []
+        version_diffs_dict["files_only_in_git"] = []
+        version_diffs_dict["files_with_different_content"] = []
+
+        only_dl, only_git, value_diff = dict_diff(
+            dl_version_files_dict, git_version_files_dict
+        )
+
+        for filepath in only_dl:
+            version_diffs_dict["files_only_in_downloads"].append(filepath)
+        for filepath in only_git:
+            version_diffs_dict["files_only_in_git"].append(filepath)
+        for filepath, _, _ in value_diff:
+            version_diffs_dict["files_with_different_content"].append(filepath)
+
+        diffs_dict["differences"][version] = version_diffs_dict
+
+        if len(only_dl) > 0 or len(value_diff) > 0:
+            nonmatching_versions.append(version)
+            extraneous_count += len(only_dl)
+            differing_count += len(value_diff)
+
+    nonmatching_versions.sort(key=lambda v: version_as_comparable_list(v), reverse=True)
+
+    # fully sort differences.json:
+    working_dict = diffs_dict["differences"]
+    # sort filepaths of each version, alphabetically
+    for k in working_dict.keys():
+        working_dict[k]["files_only_in_downloads"].sort()
+        working_dict[k]["files_only_in_git"].sort()
+        working_dict[k]["files_with_different_content"].sort()
+    # sort version numbers, in decreasing order
+    diffs_dict["differences"] = OrderedDict(
+        sorted(
+            working_dict.items(),
+            key=lambda p: version_as_comparable_list(p[0]),
+            reverse=True,
+        )
+    )
+
+    write_json("differences.json", diffs_dict)
+
+    if len(nonmatching_versions) > 0:
+        raise CFBSExitError(
+            "The masterfiles downloaded from github.com and cfengine.com do not match - found "
+            + str(extraneous_count)
+            + " extraneous file"
+            + ("" if extraneous_count == 1 else "s")
+            + " and "
+            + str(differing_count)
+            + " differing file"
+            + ("" if differing_count == 1 else "s")
+            + " across "
+            + str(len(nonmatching_versions))
+            + " version"
+            + ("" if len(nonmatching_versions) == 1 else "s")
+            + " ("
+            + ", ".join(nonmatching_versions)
+            + "). See ./differences.json"
+        )