From d83058e40b66c8a5bdafc6d8dab4b4029a9526a8 Mon Sep 17 00:00:00 2001 From: Victor Moene Date: Mon, 12 Jan 2026 17:53:14 +0100 Subject: [PATCH] Removed generate-release-information command Ticket: ENT-13632 Signed-off-by: Victor Moene --- README.md | 1 - cfbs/args.py | 18 --- cfbs/commands.py | 11 +- cfbs/{masterfiles => }/download.py | 85 +------------- cfbs/main.py | 25 ---- cfbs/masterfiles/__init__.py | 0 cfbs/masterfiles/analyze.py | 93 --------------- .../masterfiles/check_download_matches_git.py | 108 ------------------ .../generate_release_information.py | 50 -------- cfbs/masterfiles/generate_vcf_download.py | 33 ------ cfbs/masterfiles/generate_vcf_git_checkout.py | 102 ----------------- 11 files changed, 2 insertions(+), 524 deletions(-) rename cfbs/{masterfiles => }/download.py (61%) delete mode 100644 cfbs/masterfiles/__init__.py delete mode 100644 cfbs/masterfiles/analyze.py delete mode 100644 cfbs/masterfiles/check_download_matches_git.py delete mode 100644 cfbs/masterfiles/generate_release_information.py delete mode 100644 cfbs/masterfiles/generate_vcf_download.py delete mode 100644 cfbs/masterfiles/generate_vcf_git_checkout.py diff --git a/README.md b/README.md index 4337b55f..9b468241 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,6 @@ These commands are intended to be run as part of build systems / deployment pipe `cfbs set-input` and `cfbs get-input` can be thought of as ways to save and load the input file. Similar to `cfbs get-input` the JSON contains both the specification (what the module accepts and how it's presented to the user) as well as the user's responses (if present). Expected usage is to run `cfbs get-input` to get the JSON, and then fill out the response part and run `cfbs set-input`. -- `cfbs generate-release-information`: An internal command used to generate JSON release information files from the [official CFEngine masterfiles](https://github.com/cfengine/masterfiles/). - `cfbs validate`: Used to validate the [index JSON file](https://github.com/cfengine/build-index/blob/master/cfbs.json). May be expanded to validate other files and formats in the future. **Note:** If you use `cfbs validate` as part of your automation, scripts, and build systems, be aware that we might add more strict validation rules in the future, so be prepared to sometimes have it fail after upgrading the version of cfbs. diff --git a/cfbs/args.py b/cfbs/args.py index d46d6277..ef9dc76c 100644 --- a/cfbs/args.py +++ b/cfbs/args.py @@ -28,9 +28,6 @@ class ArgsTypesNamespace(argparse.Namespace): git_commit_message = None # type: Optional[str] ignore_versions_json = False # type: bool diffs = None # type: Optional[str] - omit_download = False # type: bool - check_against_git = False # type: bool - minimum_version = None # type: Optional[str] to_json = None # type: Optional[str] reference_version = None # type: Optional[str] masterfiles_dir = None # type: Optional[str] @@ -169,21 +166,6 @@ def get_arg_parser(whitespace_for_manual=False): const="diffs.txt", default=None, ) - parser.add_argument( - "--omit-download", - help="Use existing masterfiles instead of downloading in 'cfbs generate-release-information'", - action="store_true", - ) - parser.add_argument( - "--check-against-git", - help="Check whether masterfiles from cfengine.com and github.com match in 'cfbs generate-release-information'", - action="store_true", - ) - parser.add_argument( - "--from", - help="Specify minimum version in 'cfbs generate-release-information'", - dest="minimum_version", - ) parser.add_argument( "--to-json", help="Output 'cfbs analyze' results to a JSON file; optionally specify the JSON's filename", diff --git a/cfbs/commands.py b/cfbs/commands.py index 279a2801..fbeb277c 100644 --- a/cfbs/commands.py +++ b/cfbs/commands.py @@ -59,7 +59,7 @@ def search_command(terms: List[str]): from cfbs.cfbs_json import CFBSJson from cfbs.cfbs_types import CFBSCommandExitCode, CFBSCommandGitResult -from cfbs.masterfiles.download import download_single_version +from cfbs.download import download_single_version from cfbs.updates import ModuleUpdates, update_module from cfbs.utils import ( CFBSNetworkError, @@ -123,7 +123,6 @@ def search_command(terms: List[str]): from cfbs.git_magic import commit_after_command, git_commit_maybe_prompt from cfbs.prompts import prompt_user, prompt_user_yesno from cfbs.module import Module, is_module_absolute, is_module_added_manually -from cfbs.masterfiles.generate_release_information import generate_release_information _MODULES_URL = "https://archive.build.cfengine.com/modules" @@ -1610,11 +1609,3 @@ def get_input_command(name, outfile): log.error("Failed to write json: %s" % e) return 1 return 0 - - -@cfbs_command("generate-release-information") -def generate_release_information_command( - omit_download=False, check=False, min_version=None -): - generate_release_information(omit_download, check, min_version) - return 0 diff --git a/cfbs/masterfiles/download.py b/cfbs/download.py similarity index 61% rename from cfbs/masterfiles/download.py rename to cfbs/download.py index 388f2e0a..9df20ffe 100644 --- a/cfbs/masterfiles/download.py +++ b/cfbs/download.py @@ -1,14 +1,7 @@ import os import shutil -from cfbs.utils import ( - CFBSNetworkError, - fetch_url, - get_json, - mkdir, - CFBSExitError, - version_is_at_least, -) +from cfbs.utils import CFBSNetworkError, fetch_url, get_json, mkdir, CFBSExitError ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json" @@ -40,72 +33,6 @@ } -def get_download_urls_enterprise(min_version=None): - download_urls = {} - reported_checksums = {} - - print("* gathering download URLs...") - - try: - data = get_json(ENTERPRISE_RELEASES_URL) - except CFBSNetworkError: - raise CFBSExitError( - "Downloading CFEngine release data failed - check your Wi-Fi / network settings." - ) - - for release_data in data["releases"]: - version = release_data["version"] - - if not version_is_at_least(version, min_version): - continue - - if version in MISSING_DATA_VERSIONS: - download_urls[version] = HARDCODED_URLS[version] - reported_checksums[version] = HARDCODED_CHECKSUMS[version] - continue - - release_url = release_data["URL"] - try: - subdata = get_json(release_url) - except CFBSNetworkError: - raise CFBSExitError( - "Downloading CFEngine release data for version %s failed - check your Wi-Fi / network settings." - % version - ) - artifacts_data = subdata["artifacts"] - - if "Additional Assets" not in artifacts_data: - # happens for 3.9.0b1, 3.8.0b1, 3.6.1, 3.6.0 - continue - - masterfiles_data = None - for asset in artifacts_data["Additional Assets"]: - if asset["Title"] == "Masterfiles ready-to-install tarball": - masterfiles_data = asset - - if masterfiles_data is None: - # happens for 3.9.2, 3.9.0, 3.8.2, 3.8.1, 3.8.0, 3.7.4--3.6.2 - # 3.9.2: see above - # 3.9.0 and below: no masterfiles listed, and unlisted analogous URLs seemingly do not exist - continue - - download_urls[version] = masterfiles_data["URL"] - reported_checksums[version] = masterfiles_data["SHA256"] - - return download_urls, reported_checksums - - -def get_all_download_urls(min_version=None): - download_urls, reported_checksums = get_download_urls_enterprise(min_version) - - for version in COMMUNITY_ONLY_VERSIONS: - if version_is_at_least(version, min_version): - download_urls[version] = HARDCODED_URLS[version] - reported_checksums[version] = HARDCODED_CHECKSUMS[version] - - return download_urls, reported_checksums - - def get_single_download_url(version): if version in HARDCODED_VERSIONS: download_url = HARDCODED_URLS[version] @@ -177,16 +104,6 @@ def download_versions_from_urls(download_path, download_urls, reported_checksums return downloaded_versions -def download_all_versions(download_path, min_version=None): - download_urls, reported_checksums = get_all_download_urls(min_version) - - downloaded_versions = download_versions_from_urls( - download_path, download_urls, reported_checksums - ) - - return downloaded_versions - - def download_single_version(download_path, version): download_url, reported_checksum = get_single_download_url(version) diff --git a/cfbs/main.py b/cfbs/main.py index f2f1b35b..72f14407 100644 --- a/cfbs/main.py +++ b/cfbs/main.py @@ -90,24 +90,6 @@ def _main() -> int: % args.command ) - if args.omit_download and args.command != "generate-release-information": - raise CFBSUserError( - "The option --omit-download is only for 'cfbs generate-release-information', not 'cfbs %s'" - % args.command - ) - - if args.check_against_git and args.command != "generate-release-information": - raise CFBSUserError( - "The option --check-against-git is only for 'cfbs generate-release-information', not 'cfbs %s'" - % args.command - ) - - if args.minimum_version and args.command != "generate-release-information": - raise CFBSUserError( - "The option --from is only for 'cfbs generate-release-information', not 'cfbs %s'" - % args.command - ) - if args.masterfiles_dir and args.command not in ("analyze", "analyse"): raise CFBSUserError( "The option --masterfiles-dir is only for 'cfbs analyze', not 'cfbs %s'" @@ -193,13 +175,6 @@ def _main() -> int: if args.command == "convert": return commands.convert_command(args.non_interactive, args.offline) - if args.command == "generate-release-information": - return commands.generate_release_information_command( - omit_download=args.omit_download, - check=args.check_against_git, - min_version=args.minimum_version, - ) - # Commands you cannot run outside a cfbs repo: if not is_cfbs_repo(): raise CFBSExitError("This is not a cfbs repo, to get started, type: cfbs init") diff --git a/cfbs/masterfiles/__init__.py b/cfbs/masterfiles/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cfbs/masterfiles/analyze.py b/cfbs/masterfiles/analyze.py deleted file mode 100644 index b56da27e..00000000 --- a/cfbs/masterfiles/analyze.py +++ /dev/null @@ -1,93 +0,0 @@ -from collections import OrderedDict -import os - -from cfbs.utils import dict_sorted_by_key, file_sha256, version_as_comparable_list - -Version = str - - -def initialize_vcf(): - versions_dict = {"versions": {}} - checksums_dict = {"checksums": {}} - files_dict = {"files": {}} - - return versions_dict, checksums_dict, files_dict - - -def versions_checksums_files( - files_dir_path, version, versions_dict, checksums_dict, files_dict -): - for root, _, files in os.walk(files_dir_path): - for name in files: - full_relpath = os.path.join(root, name) - tarball_relpath = os.path.relpath(full_relpath, files_dir_path) - file_checksum = file_sha256(full_relpath) - - if version not in versions_dict["versions"]: - versions_dict["versions"][version] = {} - versions_dict["versions"][version][tarball_relpath] = file_checksum - - if file_checksum not in checksums_dict["checksums"]: - checksums_dict["checksums"][file_checksum] = {} - if tarball_relpath not in checksums_dict["checksums"][file_checksum]: - checksums_dict["checksums"][file_checksum][tarball_relpath] = [] - checksums_dict["checksums"][file_checksum][tarball_relpath].append(version) - - if tarball_relpath not in files_dict["files"]: - files_dict["files"][tarball_relpath] = {} - if file_checksum not in files_dict["files"][tarball_relpath]: - files_dict["files"][tarball_relpath][file_checksum] = [] - files_dict["files"][tarball_relpath][file_checksum].append(version) - - return versions_dict, checksums_dict, files_dict - - -def finalize_vcf(versions_dict, checksums_dict, files_dict): - # explicitly sort VCF data to ensure determinism - - # checksums.json: - working_dict = checksums_dict["checksums"] - for c in working_dict.keys(): - for f in working_dict[c].keys(): - # sort each version list, descending - working_dict[c][f] = sorted( - working_dict[c][f], - key=lambda v: version_as_comparable_list(v), - reverse=True, - ) - # sort filepaths, alphabetically - working_dict[c] = dict_sorted_by_key(working_dict[c]) - # sort checksums - checksums_dict["checksums"] = dict_sorted_by_key(working_dict) - - # files.json: - working_dict = files_dict["files"] - # sort each list, first by version descending, then by checksum - for f in working_dict.keys(): - for c in working_dict[f].keys(): - # sort each version list, descending - working_dict[f][c] = sorted( - working_dict[f][c], - key=lambda v: version_as_comparable_list(v), - reverse=True, - ) - # sort checksums - working_dict[f] = dict_sorted_by_key(working_dict[f]) - # sort files, alphabetically - files_dict["files"] = dict_sorted_by_key(working_dict) - - # versions.json: - working_dict = versions_dict["versions"] - # sort files of each version - for v in working_dict.keys(): - working_dict[v] = dict_sorted_by_key(working_dict[v]) - # sort version numbers, in decreasing order - versions_dict["versions"] = OrderedDict( - sorted( - working_dict.items(), - key=lambda p: version_as_comparable_list(p[0]), - reverse=True, - ) - ) - - return versions_dict, checksums_dict, files_dict diff --git a/cfbs/masterfiles/check_download_matches_git.py b/cfbs/masterfiles/check_download_matches_git.py deleted file mode 100644 index 0e54e7d8..00000000 --- a/cfbs/masterfiles/check_download_matches_git.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -from collections import OrderedDict - -from cfbs.utils import ( - dict_diff, - read_json, - CFBSExitError, - write_json, - version_as_comparable_list, -) - - -def check_download_matches_git(versions): - """Check that the downloadable files match the git files. - - This can be used to monitor / detect if something has been changed, accidentally or maliciously. - - Generates a `differences-*.txt` file for each version. - """ - assert os.path.isfile("versions.json") - assert os.path.isfile("versions-git.json") - - download_versions_dict = read_json("versions.json") - git_versions_dict = read_json("versions-git.json") - - assert download_versions_dict is not None - assert git_versions_dict is not None - - diffs_dict = {"differences": {}} - - nonmatching_versions = [] - extraneous_count = 0 - differing_count = 0 - - for version in versions: - dl_version_files_dict = download_versions_dict["versions"][version] - git_version_files_dict = git_versions_dict["versions"][version] - - # normalize downloaded version dictionary filepaths - # necessary because the downloaded version and git version dictionaries have filepaths of different forms - new_download_dict = {} - for key, value in dl_version_files_dict.items(): - if key.startswith("masterfiles/"): - key = key[12:] - new_download_dict[key] = value - dl_version_files_dict = new_download_dict - - version_diffs_dict = {} - version_diffs_dict["files_only_in_downloads"] = [] - version_diffs_dict["files_only_in_git"] = [] - version_diffs_dict["files_with_different_content"] = [] - - only_dl, only_git, value_diff = dict_diff( - dl_version_files_dict, git_version_files_dict - ) - - for filepath in only_dl: - version_diffs_dict["files_only_in_downloads"].append(filepath) - for filepath in only_git: - version_diffs_dict["files_only_in_git"].append(filepath) - for filepath, _, _ in value_diff: - version_diffs_dict["files_with_different_content"].append(filepath) - - diffs_dict["differences"][version] = version_diffs_dict - - if len(only_dl) > 0 or len(value_diff) > 0: - nonmatching_versions.append(version) - extraneous_count += len(only_dl) - differing_count += len(value_diff) - - nonmatching_versions.sort(key=lambda v: version_as_comparable_list(v), reverse=True) - - # fully sort differences.json: - working_dict = diffs_dict["differences"] - # sort filepaths of each version, alphabetically - for k in working_dict.keys(): - working_dict[k]["files_only_in_downloads"].sort() - working_dict[k]["files_only_in_git"].sort() - working_dict[k]["files_with_different_content"].sort() - # sort version numbers, in decreasing order - diffs_dict["differences"] = OrderedDict( - sorted( - working_dict.items(), - key=lambda p: version_as_comparable_list(p[0]), - reverse=True, - ) - ) - - write_json("differences.json", diffs_dict) - - if len(nonmatching_versions) > 0: - raise CFBSExitError( - "The masterfiles downloaded from github.com and cfengine.com do not match - found " - + str(extraneous_count) - + " extraneous file" - + ("" if extraneous_count == 1 else "s") - + " and " - + str(differing_count) - + " differing file" - + ("" if differing_count == 1 else "s") - + " across " - + str(len(nonmatching_versions)) - + " version" - + ("" if len(nonmatching_versions) == 1 else "s") - + " (" - + ", ".join(nonmatching_versions) - + "). See ./differences.json" - ) diff --git a/cfbs/masterfiles/generate_release_information.py b/cfbs/masterfiles/generate_release_information.py deleted file mode 100644 index d01dae5c..00000000 --- a/cfbs/masterfiles/generate_release_information.py +++ /dev/null @@ -1,50 +0,0 @@ -from cfbs.masterfiles.download import download_all_versions -from cfbs.masterfiles.generate_vcf_download import generate_vcf_download -from cfbs.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout -from cfbs.masterfiles.check_download_matches_git import check_download_matches_git -from cfbs.utils import immediate_subdirectories, version_is_at_least - -DOWNLOAD_PATH = "downloaded_masterfiles" - - -def generate_release_information(omit_download=False, check=False, min_version=None): - if not omit_download: - print("Downloading masterfiles...") - - downloaded_versions = download_all_versions(DOWNLOAD_PATH, min_version) - - print("Download finished. Every reported checksum matches.") - else: - downloaded_versions = immediate_subdirectories(DOWNLOAD_PATH) - - downloaded_versions = list( - filter( - lambda v: version_is_at_least(v, min_version), - downloaded_versions, - ) - ) - - print( - "Downloading releases of masterfiles from cfengine.com and generating release information..." - ) - generate_vcf_download(DOWNLOAD_PATH, downloaded_versions) - - if check: - print( - "Downloading releases of masterfiles from git (github.com) and generating " - + "additional release information for comparison..." - ) - generate_vcf_git_checkout(downloaded_versions) - print("Candidate release information generated.") - print("Comparing files from cfengine.com and github.com...") - - check_download_matches_git(downloaded_versions) - - print("The masterfiles downloaded from github.com and cfengine.com match.") - else: - print("Release information successfully generated.") - print("See the results in ./masterfiles/") - print( - "(Run again with --check-against-git to download and compare with files " - + "from git, and generate -git.json files)" - ) diff --git a/cfbs/masterfiles/generate_vcf_download.py b/cfbs/masterfiles/generate_vcf_download.py deleted file mode 100644 index c65c2c65..00000000 --- a/cfbs/masterfiles/generate_vcf_download.py +++ /dev/null @@ -1,33 +0,0 @@ -import os - -from cfbs.utils import write_json -from cfbs.masterfiles.analyze import ( - finalize_vcf, - initialize_vcf, - versions_checksums_files, -) - - -def generate_vcf_download(dir_path, downloaded_versions): - """`dir_path`: the path of the directory containing masterfiles versions - subdirectories in the form `dir_path/x.y.z/tarball/` - - The `tarball` folder should contain the `masterfiles` folder (older - tarballs also have a `modules` folder alongside the `masterfiles` folder). - """ - versions_dict, checksums_dict, files_dict = initialize_vcf() - - for version in downloaded_versions: - files_dir_path = os.path.join(dir_path, version, "tarball") - - versions_dict, checksums_dict, files_dict = versions_checksums_files( - files_dir_path, version, versions_dict, checksums_dict, files_dict - ) - - versions_dict, checksums_dict, files_dict = finalize_vcf( - versions_dict, checksums_dict, files_dict - ) - - write_json("./masterfiles/versions.json", versions_dict) - write_json("./masterfiles/checksums.json", checksums_dict) - write_json("./masterfiles/files.json", files_dict) diff --git a/cfbs/masterfiles/generate_vcf_git_checkout.py b/cfbs/masterfiles/generate_vcf_git_checkout.py deleted file mode 100644 index 9a1ffa8e..00000000 --- a/cfbs/masterfiles/generate_vcf_git_checkout.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -import shutil -import subprocess -import sys - -from cfbs.utils import write_json -from cfbs.masterfiles.analyze import ( - finalize_vcf, - initialize_vcf, - versions_checksums_files, -) - -DIR_PATH = "." -"""The path of the working directory.""" - -MPF_URL = "https://github.com/cfengine/masterfiles" -MPF_PATH = os.path.join(DIR_PATH, "masterfiles") - - -def check_required_command(command): - if not shutil.which(command): - print("`%s` was not found" % command) - sys.exit(1) - - -def check_required_commands(commands): - for c in commands: - check_required_command(c) - - -def generate_vcf_git_checkout(checkout_tags): - required_commands = ["git", "make", "automake", "autoconf"] - check_required_commands(required_commands) - - # get the current version of the MPF repo - if not os.path.isdir(MPF_PATH): - subprocess.run( - ["git", "clone", "--no-checkout", MPF_URL], - cwd=DIR_PATH, - check=True, - ) - else: - subprocess.run( - ["git", "fetch", "--all", "--tags", "--force"], - cwd=MPF_PATH, - check=True, - ) - - versions_dict, checksums_dict, files_dict = initialize_vcf() - - for tag in checkout_tags: - print("Checking out tag", tag) - - # check out the version - subprocess.run( - ["git", "checkout", tag], - cwd=MPF_PATH, - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - # build masterfiles from git as they are in the tarball packages - # for the files of this version to be reproducible, the `EXPLICIT_RELEASE` - # environment variable needs to be set to what it was when the downloadable - # files were built - if tag == "3.18.3": - release_number = "2" - else: - release_number = "1" - subprocess.run( - ["./autogen.sh"], - cwd=MPF_PATH, - check=True, - env=dict( - os.environ.copy(), EXPLICIT_VERSION=tag, EXPLICIT_RELEASE=release_number - ), - ) - # older masterfiles version READMEs instruct to use `make install` and newer `make` - always use `make` instead - subprocess.run(["make"], cwd=MPF_PATH, check=True) - - # compute VCF data for all the files - versions_dict, checksums_dict, files_dict = versions_checksums_files( - MPF_PATH, tag, versions_dict, checksums_dict, files_dict - ) - - # clean the files to prevent spillage to other versions - subprocess.run( - ["git", "clean", "-dfx"], - cwd=MPF_PATH, - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - versions_dict, checksums_dict, files_dict = finalize_vcf( - versions_dict, checksums_dict, files_dict - ) - - write_json("versions-git.json", versions_dict) - write_json("checksums-git.json", checksums_dict) - write_json("files-git.json", files_dict)