Merge pull request #297 from victormlg/generate_release

 ENT-13632: Removed generate-release-information command

Author: larsewi

README.md

@@ -293,7 +293,6 @@ These commands are intended to be run as part of build systems / deployment pipe
   `cfbs set-input` and `cfbs get-input` can be thought of as ways to save and load the input file.
   Similar to `cfbs get-input` the JSON contains both the specification (what the module accepts and how it's presented to the user) as well as the user's responses (if present).
   Expected usage is to run `cfbs get-input` to get the JSON, and then fill out the response part and run `cfbs set-input`.
-- `cfbs generate-release-information`: An internal command used to generate JSON release information files from the [official CFEngine masterfiles](https://github.com/cfengine/masterfiles/).
 - `cfbs validate`: Used to validate the [index JSON file](https://github.com/cfengine/build-index/blob/master/cfbs.json).
   May be expanded to validate other files and formats in the future.
   **Note:** If you use `cfbs validate` as part of your automation, scripts, and build systems, be aware that we might add more strict validation rules in the future, so be prepared to sometimes have it fail after upgrading the version of cfbs.

cfbs/args.py

@@ -28,9 +28,6 @@ class ArgsTypesNamespace(argparse.Namespace):
     git_commit_message = None  # type: Optional[str]
     ignore_versions_json = False  # type: bool
     diffs = None  # type: Optional[str]
-    omit_download = False  # type: bool
-    check_against_git = False  # type: bool
-    minimum_version = None  # type: Optional[str]
     to_json = None  # type: Optional[str]
     reference_version = None  # type: Optional[str]
     masterfiles_dir = None  # type: Optional[str]
@@ -169,21 +166,6 @@ def get_arg_parser(whitespace_for_manual=False):
         const="diffs.txt",
         default=None,
     )
-    parser.add_argument(
-        "--omit-download",
-        help="Use existing masterfiles instead of downloading in 'cfbs generate-release-information'",
-        action="store_true",
-    )
-    parser.add_argument(
-        "--check-against-git",
-        help="Check whether masterfiles from cfengine.com and github.com match in 'cfbs generate-release-information'",
-        action="store_true",
-    )
-    parser.add_argument(
-        "--from",
-        help="Specify minimum version in 'cfbs generate-release-information'",
-        dest="minimum_version",
-    )
     parser.add_argument(
         "--to-json",
         help="Output 'cfbs analyze' results to a JSON file; optionally specify the JSON's filename",

cfbs/commands.py

@@ -59,7 +59,7 @@ def search_command(terms: List[str]):
 
 from cfbs.cfbs_json import CFBSJson
 from cfbs.cfbs_types import CFBSCommandExitCode, CFBSCommandGitResult
-from cfbs.masterfiles.download import download_single_version
+from cfbs.download import download_single_version
 from cfbs.updates import ModuleUpdates, update_module
 from cfbs.utils import (
     CFBSNetworkError,
@@ -123,7 +123,6 @@ def search_command(terms: List[str]):
 from cfbs.git_magic import commit_after_command, git_commit_maybe_prompt
 from cfbs.prompts import prompt_user, prompt_user_yesno
 from cfbs.module import Module, is_module_absolute, is_module_added_manually
-from cfbs.masterfiles.generate_release_information import generate_release_information
 
 _MODULES_URL = "https://archive.build.cfengine.com/modules"
 
@@ -1610,11 +1609,3 @@ def get_input_command(name, outfile):
         log.error("Failed to write json: %s" % e)
         return 1
     return 0
-
-
-@cfbs_command("generate-release-information")
-def generate_release_information_command(
-    omit_download=False, check=False, min_version=None
-):
-    generate_release_information(omit_download, check, min_version)
-    return 0

cfbs/masterfiles/download.py cfbs/download.pycfbs/masterfiles/download.py renamed to cfbs/download.py

@@ -1,14 +1,7 @@
 import os
 import shutil
 
-from cfbs.utils import (
-    CFBSNetworkError,
-    fetch_url,
-    get_json,
-    mkdir,
-    CFBSExitError,
-    version_is_at_least,
-)
+from cfbs.utils import CFBSNetworkError, fetch_url, get_json, mkdir, CFBSExitError
 
 ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json"
 
@@ -40,72 +33,6 @@
 }
 
 
-def get_download_urls_enterprise(min_version=None):
-    download_urls = {}
-    reported_checksums = {}
-
-    print("* gathering download URLs...")
-
-    try:
-        data = get_json(ENTERPRISE_RELEASES_URL)
-    except CFBSNetworkError:
-        raise CFBSExitError(
-            "Downloading CFEngine release data failed - check your Wi-Fi / network settings."
-        )
-
-    for release_data in data["releases"]:
-        version = release_data["version"]
-
-        if not version_is_at_least(version, min_version):
-            continue
-
-        if version in MISSING_DATA_VERSIONS:
-            download_urls[version] = HARDCODED_URLS[version]
-            reported_checksums[version] = HARDCODED_CHECKSUMS[version]
-            continue
-
-        release_url = release_data["URL"]
-        try:
-            subdata = get_json(release_url)
-        except CFBSNetworkError:
-            raise CFBSExitError(
-                "Downloading CFEngine release data for version %s failed - check your Wi-Fi / network settings."
-                % version
-            )
-        artifacts_data = subdata["artifacts"]
-
-        if "Additional Assets" not in artifacts_data:
-            # happens for 3.9.0b1, 3.8.0b1, 3.6.1, 3.6.0
-            continue
-
-        masterfiles_data = None
-        for asset in artifacts_data["Additional Assets"]:
-            if asset["Title"] == "Masterfiles ready-to-install tarball":
-                masterfiles_data = asset
-
-        if masterfiles_data is None:
-            # happens for 3.9.2, 3.9.0, 3.8.2, 3.8.1, 3.8.0, 3.7.4--3.6.2
-            # 3.9.2: see above
-            # 3.9.0 and below: no masterfiles listed, and unlisted analogous URLs seemingly do not exist
-            continue
-
-        download_urls[version] = masterfiles_data["URL"]
-        reported_checksums[version] = masterfiles_data["SHA256"]
-
-    return download_urls, reported_checksums
-
-
-def get_all_download_urls(min_version=None):
-    download_urls, reported_checksums = get_download_urls_enterprise(min_version)
-
-    for version in COMMUNITY_ONLY_VERSIONS:
-        if version_is_at_least(version, min_version):
-            download_urls[version] = HARDCODED_URLS[version]
-            reported_checksums[version] = HARDCODED_CHECKSUMS[version]
-
-    return download_urls, reported_checksums
-
-
 def get_single_download_url(version):
     if version in HARDCODED_VERSIONS:
         download_url = HARDCODED_URLS[version]
@@ -177,16 +104,6 @@ def download_versions_from_urls(download_path, download_urls, reported_checksums
     return downloaded_versions
 
 
-def download_all_versions(download_path, min_version=None):
-    download_urls, reported_checksums = get_all_download_urls(min_version)
-
-    downloaded_versions = download_versions_from_urls(
-        download_path, download_urls, reported_checksums
-    )
-
-    return downloaded_versions
-
-
 def download_single_version(download_path, version):
     download_url, reported_checksum = get_single_download_url(version)
 

cfbs/main.py

@@ -90,24 +90,6 @@ def _main() -> int:
             % args.command
         )
 
-    if args.omit_download and args.command != "generate-release-information":
-        raise CFBSUserError(
-            "The option --omit-download is only for 'cfbs generate-release-information', not 'cfbs %s'"
-            % args.command
-        )
-
-    if args.check_against_git and args.command != "generate-release-information":
-        raise CFBSUserError(
-            "The option --check-against-git is only for 'cfbs generate-release-information', not 'cfbs %s'"
-            % args.command
-        )
-
-    if args.minimum_version and args.command != "generate-release-information":
-        raise CFBSUserError(
-            "The option --from is only for 'cfbs generate-release-information', not 'cfbs %s'"
-            % args.command
-        )
-
     if args.masterfiles_dir and args.command not in ("analyze", "analyse"):
         raise CFBSUserError(
             "The option --masterfiles-dir is only for 'cfbs analyze', not 'cfbs %s'"
@@ -193,13 +175,6 @@ def _main() -> int:
     if args.command == "convert":
         return commands.convert_command(args.non_interactive, args.offline)
 
-    if args.command == "generate-release-information":
-        return commands.generate_release_information_command(
-            omit_download=args.omit_download,
-            check=args.check_against_git,
-            min_version=args.minimum_version,
-        )
-
     # Commands you cannot run outside a cfbs repo:
     if not is_cfbs_repo():
         raise CFBSExitError("This is not a cfbs repo, to get started, type: cfbs init")