diff --git a/.gitignore b/.gitignore index 3ed58d8..355d9d3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,10 @@ releases/ sha512sums/ tmp/ web/ + +# Editors +*.swp + +# Python +*.pyc +__pycache__/ diff --git a/builder/__init__.py b/builder/__init__.py new file mode 100644 index 0000000..8acbd37 --- /dev/null +++ b/builder/__init__.py @@ -0,0 +1,5 @@ +from .builder import PodmanRunner, GitRunner +from .images import configs as ImageConfigs +from .config import Config, write_config, load_config + +BuildConfigs = [] # TODO diff --git a/builder/builder.py b/builder/builder.py new file mode 100644 index 0000000..fc3c792 --- /dev/null +++ b/builder/builder.py @@ -0,0 +1,163 @@ +#!/usr/python + +import logging +import os +import subprocess +import sys + +from .config import Config +from .runner import RunError, Runner, run_simple + + +def which(what): + return run_simple(["which", what]).stdout.strip() + + +def ensure_dir(dirname): + os.makedirs(dirname, exist_ok=True) + + +class PodmanRunner(Runner): + IMAGES = ["mono-glue", "windows", "ubuntu-64", "ubuntu-32", "javascript"] + IMAGES_PRIVATE = ["macosx", "android", "ios", "uwp"] + + @staticmethod + def get_images(): + return PodmanRunner.IMAGES + PodmanRunner.IMAGES_PRIVATE + + def get_image_path(self, image, version="latest", local=False): + if local: + return f"localhost/{image}:{version}" + path = Config.private_path if image in PodmanRunner.IMAGES_PRIVATE else Config.public_path + return f"{Config.registry}/{path}/{image}:{version}" + + def __init__(self, base_dir, dry_run=False): + self.base_dir = base_dir + self.dry_run = dry_run + self.logged_in = False + self._podman = self._detect_podman() + + def _detect_podman(self): + podman = which("podman") + if not podman: + podman = which("docker") + if not podman: + print("Either podman or docker needs to be installed") + sys.exit(1) + return podman + + def login(self): + if Config.username == "" or Config.password == "": + logging.debug("Skipping login, missing username or password") + return + self.logged_in = run_simple(self._podman, "login", Config.regitry, "-u", Config.username, "-p", Config.password).returncode == 0 + + def image_exists(self, image): + return run_simple([self._podman, "image", "exists", image]).returncode == 0 + + def fetch_image(self, image, force=False): + exists = not force and self.image_exists(image) + if not exists: + self.run([self._podman, "pull", "%s/%s" % (Config.registry, image)]) + + def fetch_images(self, images=[], **kwargs): + if len(images) == 0: + images = PodmanRunner.get_images() + for image in images: + if image in PodmanRunner.IMAGES: + self.fetch_image("%s/%s" % (Config.public_path, image), **kwargs) + elif image in PodmanRunner.IMAGES_PRIVATE: + if not self.logged_in: + print("Can't fetch image: %s. Not logged in" % image) + continue + self.fetch_image("%s/%s" % (Config.private_path, image), **kwargs) + + def podrun(self, run_config, classical=False, mono=False, local=False, interactive=False, **kwargs): + def env(env_vars): + for k, v in env_vars.items(): + yield("--env") + yield(f"{k}={v}") + + def mount(mount_points): + for k, v in mount_points.items(): + yield("-v") + yield(f"{self.base_dir}/{k}:/root/{v}") + + for d in run_config.dirs: + ensure_dir(os.path.join(self.base_dir, d)) + + cores = os.environ.get('NUM_CORES', os.cpu_count()) + cmd = [self._podman, "run", "--rm", "-w", "/root/"] + cmd += env({ + "BUILD_NAME": os.environ.get("BUILD_NAME", "custom_build"), + "NUM_CORES": os.environ.get("NUM_CORES", os.cpu_count()), + "CLASSICAL": 1 if classical else 0, + "MONO": 1 if mono else 0, + }) + cmd += mount({ + "mono-glue": "mono-glue", + "godot.tar.gz": "godot.tar.gz", + }) + cmd += mount(run_config.mounts) + if run_config.out_dir is not None: + out_dir = f"out/{run_config.out_dir}" + ensure_dir(f"{self.base_dir}/{out_dir}") + cmd += mount({ + out_dir: "out" + }) + cmd += run_config.extra_opts + + image_path = self.get_image_path(run_config.image, version=run_config.image_version, local=local) + if interactive: + if self.dry_run: + print(" ".join(cmd + ["-it", image_path, "bash"])) + return + return subprocess.run(cmd + ["-it", image_path, "bash"]) + + cmd += [image_path] + run_config.cmd + if run_config.log and not 'log' in kwargs: + ensure_dir(f"{self.base_dir}/out/logs") + with open(os.path.join(self.base_dir, "out", "logs", run_config.log), "w") as log: + return self.run(cmd, log=log, **kwargs) + else: + return self.run(cmd, **kwargs) + + +class GitRunner(Runner): + + def __init__(self, base_dir, dry_run=False): + self.dry_run = dry_run + self.base_dir = base_dir + + def git(self, *args, can_fail=False): + return self.run(["git"] + list(args), can_fail) + + def check_version(self, godot_version): + if self.dry_run: + print("Skipping version check in dry run mode (would likely fail)") + return + import importlib.util + version_file = os.path.join("git", "version.py") + spec = importlib.util.spec_from_file_location("version", version_file) + version = importlib.util.module_from_spec(spec) + spec.loader.exec_module(version) + if hasattr(version, "patch"): + version_string = f"{version.major}.{version.minor}.{version.patch}-{version.status}" + else: + version_string = f"{version.major}.{version.minor}-{version.status}" + ok = version_string == godot_version + if not ok: + print(f"Version mismatch, expected: {godot_version}, got: {version_string}") + sys.exit(1) + + def checkout(self, ref): + repo = "https://github.com/godotengine/godot" + dest = os.path.join(self.base_dir, "git") + self.git("clone", dest, can_fail=True) + self.git("-C", dest, "fetch", "--all") + self.git("-C", dest, "checkout", "--detach", ref) + + def tgz(self, version, ref="HEAD"): + source = os.path.join(self.base_dir, "git") + dest = os.path.join(self.base_dir, "godot.tar.gz") + self.git("-C", source, "archive", f"--prefix=godot-{version}/", "-o", dest, ref) diff --git a/builder/cli.py b/builder/cli.py new file mode 100644 index 0000000..be89eec --- /dev/null +++ b/builder/cli.py @@ -0,0 +1,172 @@ +import logging, os, sys +from argparse import ArgumentParser + +from . import Config, ImageConfigs, GitRunner, PodmanRunner, write_config, load_config + +class ConfigCLI: + ACTION = "config" + HELP = "Print or save config file" + + @staticmethod + def execute(base_dir, args): + write_config(sys.stdout) + sys.stdout.write('\n') + if args.save is not None: + path = args.save if os.path.isabs(args.save) else os.path.join(base_dir, args.save) + if not path.endswith(".json"): + print("Invalid config file: %s, must be '.json'" % args.save) + sys.exit(1) + with open(path, 'w') as w: + write_config(w) + print("Saved to file: %s" % path) + + @staticmethod + def bind(parser): + parser.add_argument("-s", "--save") + +class ImageCLI: + ACTION = "fetch" + HELP = "Fetch remote build containers" + + @staticmethod + def execute(base_dir, args): + podman = PodmanRunner( + base_dir, + dry_run=args.dry_run + ) + podman.login() + podman.fetch_images( + images = args.image, + force=args.force_download + ) + + @staticmethod + def bind(parser): + parser.add_argument("-f", "--force-download", action="store_true") + parser.add_argument("-i", "--image", action="append", default=[], help="The image to fetch, all by default. Possible values: %s" % ", ".join(PodmanRunner.get_images())) + + +class GitCLI: + ACTION = "checkout" + HELP = "git checkout, version check, tar" + + @staticmethod + def execute(base_dir, args): + git = GitRunner(base_dir, dry_run=args.dry_run) + if not args.skip_checkout: + git.checkout(args.treeish) + if not args.skip_check: + git.check_version(args.godot_version) + if not args.skip_tar: + git.tgz(args.godot_version) + + @staticmethod + def bind(parser): + parser.add_argument("treeish", help="git treeish, possibly a git ref, or commit hash.", default="origin/master") + parser.add_argument("godot_version", help="godot version (e.g. 3.1-alpha5)") + parser.add_argument("-c", "--skip-checkout", action="store_true") + parser.add_argument("-t", "--skip-tar", action="store_true") + parser.add_argument("--skip-check", action="store_true") + + +class RunCLI: + ACTION = "run" + HELP = "Run the desired containers" + + CONTAINERS = [cls.__name__.replace("Config", "") for cls in ImageConfigs] + + @staticmethod + def execute(base_dir, args): + podman = PodmanRunner(base_dir, dry_run=args.dry_run) + build_mono = args.build == "all" or args.build == "mono" + build_classical = args.build == "all" or args.build == "classical" + if len(args.container) == 0: + args.container = RunCLI.CONTAINERS + to_build = [ImageConfigs[RunCLI.CONTAINERS.index(c)] for c in args.container] + for b in to_build: + podman.podrun(b, classical=build_classical, mono=build_mono, local=not args.remote, interactive=args.interactive) + + def bind(parser): + parser.add_argument("-b", "--build", choices=["all", "classical", "mono"], default="all") + parser.add_argument("-k", "--container", action="append", default=[], help="The containers to build, one of %s" % RunCLI.CONTAINERS) + parser.add_argument("-r", "--remote", help="Run with remote containers", action="store_true") + parser.add_argument("-i", "--interactive", action="store_true", help="Enter an interactive shell inside the container instead of running the default command") + + +class ReleaseCLI: + ACTION = "release" + HELP = "Make a full release cycle, git checkout, reset, version check, tar, build all" + + @staticmethod + def execute(base_dir, args): + git = GitRunner(base_dir, dry_run=args.dry_run) + podman = PodmanRunner(base_dir, dry_run=args.dry_run) + build_mono = args.build == "all" or args.build == "mono" + build_classical = args.build == "all" or args.build == "classical" + if not args.localhost and not args.skip_download: + podman.login() + podman.fetch_images( + force=args.force_download + ) + if not args.skip_git: + git.checkout(args.git) + git.check_version(args.godot_version) + git.tgz(args.godot_version) + + for b in ImageConfigs: + podman.podrun(b, classical=build_classical, mono=build_mono, local=args.localhost) + + @staticmethod + def bind(parser): + parser.add_argument("godot_version", help="godot version (e.g. 3.1-alpha5)") + parser.add_argument("-b", "--build", choices=["all", "classical", "mono"], default="all") + parser.add_argument("-s", "--skip-download", action="store_true") + parser.add_argument("-c", "--skip-git", action="store_true") + parser.add_argument("-g", "--git", help="git treeish, possibly a git ref, or commit hash.", default="origin/master") + parser.add_argument("-f", "--force-download", action="store_true") + parser.add_argument("-l", "--localhost", action="store_true") + + +class CLI: + OPTS = [(v, getattr(Config, v)) for v in dir(Config) if not v.startswith("_")] + + def add_command(self, cli): + parser = self.subparsers.add_parser(cli.ACTION, help=cli.HELP) + parser.add_argument("-n", "--dry-run", action="store_true") + parser.set_defaults(action_func=cli.execute) + cli.bind(parser) + + def __init__(self, base_dir): + self.base_dir = base_dir + self.parser = ArgumentParser() + for k,v in CLI.OPTS: + self.parser.add_argument("--%s" % k) + self.parser.add_argument("-c", "--config", help="Configuration override") + self.subparsers = self.parser.add_subparsers(dest="action", help="The requested action", required=True) + self.add_command(ConfigCLI) + self.add_command(GitCLI) + self.add_command(ImageCLI) + self.add_command(RunCLI) + self.add_command(ReleaseCLI) + + def execute(self): + args = self.parser.parse_args() + if args.config is not None: + path = args.config if os.path.isabs(args.config) else os.path.join(self.base_dir, args.config) + if not os.path.isfile(path): + print("Invalid config file: %s" % path) + sys.exit(1) + load_config(path) + for k,v in CLI.OPTS: + override = getattr(args, k) + if override is not None: + setattr(Config, k, override) + args.action_func(self.base_dir, args) + + +def main(loglevel=logging.DEBUG): + logging.basicConfig(level=loglevel) + CLI(os.getcwd()).execute() + +if __name__ == "__main__": + main() diff --git a/builder/config.py b/builder/config.py new file mode 100644 index 0000000..1e96420 --- /dev/null +++ b/builder/config.py @@ -0,0 +1,77 @@ +import json, os + +class Config: + + # Registry for build containers. + # The default registry is the one used for official Godot builds. + # Note that some of its images are private and only accessible to selected + # contributors. + # You can build your own registry with scripts at + # https://github.com/godotengine/build-containers + registry = "registry.prehensile-tales.com" + + # Registry username + username = "" + + # Registry password + password = "" + + # Public image path + public_path = "godot" + + # Private image path + private_path = "godot-private" + + # Default build name used to distinguish between official and custom builds. + build_name = "custom_build" + + # Default number of parallel cores for each build. + num_core = os.cpu_count() + + # Set up your own signing keystore and relevant details below. + # If you do not fill all SIGN_* fields, signing will be skipped. + + # Path to pkcs12 archive. + sign_keystore = "" + + # Password for the private key. + sign_password = "" + + # Name and URL of the signed application. + # Use your own when making a thirdparty build. + sign_name = "" + sign_url = "" + + # Hostname or IP address of an OSX host (Needed for signing) + # eg "user@10.1.0.10" + osx_host = "" + # ID of the Apple certificate used to sign + osx_key_id = "" + # Bundle id for the signed app + osx_bundle_id = "" + # Username/password for Apple's signing APIs (used for atltool) + apple_id = "" + apple_id_password = "" + + +def write_config(stream): + config = {} + for k in dir(Config): + if k.startswith("_"): + continue + config[k] = getattr(Config, k) + json.dump(config, stream, indent=4, sort_keys=True) + + +def load_config(path): + with open(path, 'r') as f: + d = json.load(f) + for k,v in d.items(): + if not k.startswith("_") and hasattr(Config, k): + setattr(Config, k, v) + +try: + load_config(os.path.join(os.getcwd(), 'config.json')) +except: + # No default config + pass diff --git a/builder/images.py b/builder/images.py new file mode 100644 index 0000000..d37700f --- /dev/null +++ b/builder/images.py @@ -0,0 +1,96 @@ + +class ImageConfig: + + def __getattr__(self, name): + try: + return self.__class__.getattr(name) + except AttributeError as e: + return super().__getattr__(name) + + out_dir = None + dirs = ["out"] + extra_opts = [] + cmd = ["bash", "/root/build/build.sh"] + mounts = {} + image_version = "3.3-mono-6.12.0.114" + log = None + + +class AOTCompilersConfig: + out_dir = "aot-compilers" + image = "localhost/godot-ios" + cmd = ["bash", "-c", "'cp -r /root/aot-compilers/* /root/out'"] + + +class MonoGlueConfig(ImageConfig): + dirs = ["mono-glue"] + mounts = {"build-mono-glue": "build"} + image = "godot-mono-glue" + log = "mono-glue" + + +class WindowsConfig(ImageConfig): + out_dir = "windows" + mounts = {"build-windows": "build"} + image = "godot-windows" + log = "windows" + + +class Linux64Config(ImageConfig): + out_dir = "linux/x64" + mounts = {"build-linux": "build"} + image = "godot-ubuntu-64" + log = "linux64" + +class Linux32Config(ImageConfig): + out_dir = "linux/x86" + mounts = {"build-linux": "build"} + image = "godot-ubuntu-32" + log = "linux32" + + +class JavaScriptConfig(ImageConfig): + out_dir = "javascript" + mounts = {"build-javascript": "build"} + image = "godot-javascript" + log = "javascript" + + +class MacOSXConfig(ImageConfig): + out_dir = "macosx" + mounts = {"build-macosx": "build"} + image = "godot-osx" + log = "macosx" + + +class AndroidConfig(ImageConfig): + out_dir = "android" + mounts = {"build-android": "build"} + image = "godot-android" + log = "android" + + +class IOSConfig(ImageConfig): + out_dir = "ios" + mounts = {"build-ios": "build"} + image = "godot-ios" + log = "ios" + + +class ServerConfig(ImageConfig): + out_dir = "server/x64" + mounts = {"build-server": "build"} + image = "godot-ubuntu-64" + log = "server" + + +class UWPConfig(ImageConfig): + out_dir = "uwp" + extra_opts = ["--ulimit", "nofile=32768:32768"] + cmd = ["bash", "/root/build/build.sh"] + mounts = {"build-uwp": "build"} + image = "uwp" + log = "uwp" + + +configs = ImageConfig.__subclasses__() diff --git a/builder/runner.py b/builder/runner.py new file mode 100644 index 0000000..1775b33 --- /dev/null +++ b/builder/runner.py @@ -0,0 +1,129 @@ +import sys +import time +from subprocess import PIPE, Popen, run as run_native +from threading import Thread +import logging + +try: + from Queue import Queue, Empty +except ImportError: + from queue import Queue, Empty # python 3.x + +_NIX = 'posix' in sys.builtin_module_names + + +class RunError(Exception): + + def __init__(self, cmd, exception, code, out, err, user_msg=""): + logging.info("RUNERR: %s, exception: %r" % (cmd, exception)) + self.cmd = cmd + self.exception = exception + self.code = code + self.out = out + self.err = err + self.user_msg = user_msg + + def __repr__(self): + return "RunError(cmd=%r, exception=%r, code=%r, out=%r, err=%r, user_msg=%r)" % (self.cmd, self.exception, self.code, self.out, self.err, self.user_msg) + + +def enqueue_output(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + out.close() + +def try_read(queue, log=None, flush=True): + if flush: + out = "" + r = try_read(queue, log, flush=False) + + while r != "": + out += r + r = try_read(queue, log, flush=False) + + return out + try: + line = queue.get_nowait() + + if log is not None: + log.write(line) + + return line + + except Empty: + return "" + + +def run(cmd, lock=False, log=None, errlog=None, message="", verbose=False): + proc = None + try: + proc = Popen(cmd, stdout=PIPE, stderr=PIPE, bufsize=0, close_fds=_NIX, text=True) + except Exception as e: + raise RunError(cmd, e, -1, "", "", message) + + logging.debug("RUNNING: %s, PID: %d" % (cmd, proc.pid)) + + qo = Queue() + to = Thread(target=enqueue_output, args=(proc.stdout, qo)) + to.daemon = True + to.start() + + qe = Queue() + te = Thread(target=enqueue_output, args=(proc.stderr, qe)) + te.daemon = True + te.start() + + if not lock: + return proc + + out = "" + err = "" + + try: + def consume(queue, log): + read = try_read(queue, log) + if verbose and read: + sys.stdout.write(read) + sys.stdout.flush() + return read + + while proc.poll() is None: + out += consume(qo, log) + err += consume(qe, errlog) + time.sleep(0.1) + out += consume(qo, log) + err += consume(qe, errlog) + + except Exception as e: + raise RunError(cmd, e, proc.returncode, out, err, message) + + if proc.returncode: + logging.debug("Program failed with return code: %d" % proc.returncode) + logging.debug(err) + logging.debug("Try running: %s" % " ".join(cmd)) + + return (out, err, proc.returncode) + + +class Runner: + + def run(self, cmd, can_fail=False, **kwargs): + if getattr(self, 'dry_run', False): + logging.debug("Dry run: %s" % cmd) + return + if not 'lock' in kwargs: + kwargs['lock'] = True + if not 'verbose' in kwargs: + kwargs['verbose'] = True + res = run(cmd, **kwargs) + if not can_fail and kwargs['lock'] and res[2] != 0: + print("Command failed %s" % cmd) + sys.exit(1) + return res + +def run_simple(cmd): + logging.debug("Running command: %s" % cmd) + res = run_native(cmd, stdout=PIPE, stderr=PIPE, text=True) + logging.debug(res.stdout.strip()) + logging.debug(res.stderr.strip()) + return res diff --git a/cli.py b/cli.py new file mode 100755 index 0000000..f95e49b --- /dev/null +++ b/cli.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 + +import argparse +import logging +import os +import sys + +from builder import GitRunner, PodmanRunner, ImageConfigs, BuildConfigs +from builder.cli import CLI + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + pwd = os.path.dirname(os.path.realpath(__file__)) + cli = CLI(pwd) + cli.execute()