diff --git a/robotpy_installer/_bootstrap.py b/robotpy_installer/_bootstrap.py new file mode 100644 index 0000000..b0d35e9 --- /dev/null +++ b/robotpy_installer/_bootstrap.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +description = """ +Standalone bootstrap script that can install from a bundle created with make-offline. +It will ignore your local installation and install whatever is in the bundle +""" + + +import argparse +import dataclasses +import json +import os.path +import pathlib +import shutil +import subprocess +import sys +import tempfile +import typing +import zipfile + +METADATA_VERSION = 1 +METADATA_JSON = "rpybundle.json" + + +@dataclasses.dataclass +class Metadata: + """ + Describes content of METADATA_JSON in offline bundle + """ + + #: metadata version + version: int + + #: robotpy-installer version + installer_version: str + + #: wpilib year + wpilib_year: str + + #: python ipk name + python_ipk: str + + #: python wheel tags supported by this bundle + wheel_tags: typing.List[str] + + # #: list of packages derived from pyproject.toml + # packages: typing.List[str] + + def dumps(self) -> str: + data = dataclasses.asdict(self) + return json.dumps(data) + + @classmethod + def loads(cls, s: str) -> Metadata: + data = json.loads(s) + if not isinstance(data, dict): + raise ValueError("invalid metadata") + version = data.get("version", None) + if not isinstance(version, int): + raise ValueError(f"invalid metadata version {version!r}") + if version > METADATA_VERSION: + raise ValueError( + f"can only understand metadata < {METADATA_VERSION}, got {version}" + ) + + installer_version = data.get("installer_version", None) + if not isinstance(installer_version, str): + raise ValueError(f"invalid installer version {installer_version!r}") + + wpilib_year = data.get("wpilib_year", None) + if not isinstance(wpilib_year, str): + raise ValueError(f"invalid wpilib_year {wpilib_year}") + + python_ipk = data.get("python_ipk", None) + if not python_ipk or not isinstance(python_ipk, str): + raise ValueError(f"invalid python_ipk value") + + wheel_tags = data.get("wheel_tags", None) + if not isinstance(wheel_tags, list) or len(wheel_tags) == 0: + raise ValueError(f"no wheel tags present") + # packages = data.get("packages") + # if not isinstance(packages, list): + # raise ValueError(f"invalid package list {packages!r}") + + return Metadata( + version=version, + installer_version=installer_version, + wpilib_year=wpilib_year, + python_ipk=python_ipk, + wheel_tags=wheel_tags, # packages=packages + ) + + +if __name__ == "__main__": + + # If is running from a zipfile, identify it + bundle_path = None + if pathlib.Path(__file__).parent.is_file(): + bundle_path = pathlib.Path(__file__).parent + + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + "--user", + "-u", + default=False, + action="store_true", + help="Use `pip install --user` to install packages", + ) + + if bundle_path is None: + parser.add_argument("bundle", type=pathlib.Path, help="Bundle file") + + # parser.add_argument( + # "--no-robotpy-installer", + # default=False, + # action="store_true", + # help="Do not install robotpy-installer", + # ) + args = parser.parse_args() + + if bundle_path is None: + bundle_path= args.bundle + + with zipfile.ZipFile(bundle_path, "r") as zfp: + + # extract metadata + raw_metadata = zfp.read(METADATA_JSON).decode("utf-8") + metadata = Metadata.loads(raw_metadata) + + cache_root = pathlib.Path.home() / "wpilib" / metadata.wpilib_year / "robotpy" + + # extract pip cache to a temporary directory + with tempfile.TemporaryDirectory() as t: + pip_cache = pathlib.Path(t) + + + print("Extracting wheels to temporary path...") + + for info in zfp.infolist(): + p = pathlib.Path(info.filename.replace("/", os.path.sep)) + if p.parts[0] == "pip_cache": + with zfp.open(info) as sfp, open(pip_cache / p.name, "wb") as dfp: + shutil.copyfileobj(sfp, dfp) + + # TODO: when doing local dev, how do I embed the right + # robotpy-installer? or just ignore it + + # if not args.no_robotpy_installer: + # print("Installing robotpy-installer", metadata.installer_version) + + # # install robotpy-installer offline from temporary directory + # # do == to force pip to install this one + # pip_args = [ + # sys.executable, + # "-m", + # "pip", + # "install", + # "--disable-pip-version-check", + # "--no-index", + # "--find-links", + # t, + # f"robotpy-installer=={metadata.installer_version}", + # ] + # print("+", *pip_args) + # subprocess.check_call(pip_args) + + # If this is part of robotpy-installer, on Windows need + # to worry about sharing violation + + sync_args = [ + sys.executable, + "-m", + "robotpy", + "sync", + "--from", + str(bundle_path) + ] + + if args.user: + sync_args.append("--user") + + result = subprocess.run(sync_args) + sys.exit(result.returncode) + diff --git a/robotpy_installer/_pipstub.py b/robotpy_installer/_pipstub.py index 904360c..dd08a5f 100644 --- a/robotpy_installer/_pipstub.py +++ b/robotpy_installer/_pipstub.py @@ -9,6 +9,8 @@ import os import sys +rio_python_version = "3.12.0" + if __name__ == "__main__": # Setup environment for what the RoboRIO python would have @@ -18,6 +20,6 @@ platform.machine = lambda: "roborio" platform.python_implementation = lambda: "CPython" platform.system = lambda: "Linux" - platform.python_version = lambda: "3.12.0" + platform.python_version = lambda: rio_python_version runpy.run_module("pip", run_name="__main__") diff --git a/robotpy_installer/cli_makeoffline.py b/robotpy_installer/cli_makeoffline.py new file mode 100644 index 0000000..7042cae --- /dev/null +++ b/robotpy_installer/cli_makeoffline.py @@ -0,0 +1,137 @@ +import argparse +import json +import logging +import pathlib +import shutil +import subprocess +import sys +import tempfile +import zipfile + +import packaging.tags + +from . import pyproject +from .errors import Error +from .installer import RobotpyInstaller, _WPILIB_YEAR +from .utils import handle_cli_error + +from . import _bootstrap + +logger = logging.getLogger("bundler") + + +class MakeOffline: + """ + Creates a bundle that can be used to install RobotPy dependencies + without internet access. + + To install from the bundle, + """ + + def __init__(self, parser: argparse.ArgumentParser): + parser.add_argument( + "--use-certifi", + action="store_true", + default=False, + help="Use SSL certificates from certifi", + ) + parser.add_argument( + "bundle_path", + type=pathlib.Path + ) + + @handle_cli_error + def run( + self, project_path: pathlib.Path, use_certifi: bool, bundle_path: pathlib.Path + ): + + installer = RobotpyInstaller() + # local_cache = installer.cache_root + local_pip_cache = installer.pip_cache + + bootstrap_py_path = pathlib.Path(_bootstrap.__file__) + + # collect deps from project, or use installed version + project = pyproject.load( + project_path, write_if_missing=False, default_if_missing=True + ) + packages = project.get_install_list() + + logger.info("Robot project requirements:") + for package in packages: + logger.info("- %s", package) + + # Download python ipk to original cache + python_ipk = installer.download_python(use_certifi) + + # Make temporary directory to download to + with tempfile.TemporaryDirectory() as t: + tpath = pathlib.Path(t) + installer.set_cache_root(tpath) + whl_path = tpath / "pip_cache" + + if True: + whl_path.mkdir(parents=True, exist_ok=True) + if False: + # Download rio deps (use local cache to speed it up) + installer.pip_download(False, False, [], packages, local_pip_cache) + + # Download local deps + pip_args = [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "download", + "-d", + str(whl_path), + ] + packages + + logger.debug("Using pip to download: %s", pip_args) + retval = subprocess.call(pip_args) + if retval != 0: + raise Error("pip download failed") + + from .version import version + # TODO: it's possible for the bundle to include a version of robotpy-installer + # that does not match the current version. Need to not do that. + + metadata = _bootstrap.Metadata( + version=_bootstrap.METADATA_VERSION, + installer_version=version, + wpilib_year=_WPILIB_YEAR, + python_ipk=python_ipk.name, + wheel_tags=[ + str(next(packaging.tags.sys_tags())), # sys tag + # roborio tag + ], + ) + + logger.info("Bundle supported wheel tags:") + for tag in metadata.wheel_tags: + logger.info("+ %s", tag) + + logger.info("Making bundle at '%s'", bundle_path) + + # zip it all up + with zipfile.ZipFile(bundle_path, "w") as zfp: + + # descriptor + logger.info("+ %s", _bootstrap.METADATA_JSON) + zfp.writestr(_bootstrap.METADATA_JSON, metadata.dumps()) + + # bootstrapper + logger.info("+ __main__.py") + zfp.write(bootstrap_py_path, "__main__.py") + + # ipk + logger.info("+ %s", python_ipk.name) + zfp.write(python_ipk, python_ipk.name) + + # pip cache + for f in whl_path.iterdir(): + logger.info("+ pip_cache/%s", f.name) + zfp.write(f, f"pip_cache/{f.name}") + + st = bundle_path.stat() + logger.info("Bundle is %d bytes", st.st_size) diff --git a/robotpy_installer/installer.py b/robotpy_installer/installer.py index 37bb25b..8f4fdf2 100755 --- a/robotpy_installer/installer.py +++ b/robotpy_installer/installer.py @@ -67,9 +67,7 @@ def catch_ssh_error(msg: str): class RobotpyInstaller: def __init__(self, *, log_startup: bool = True): - self.cache_root = pathlib.Path.home() / "wpilib" / _WPILIB_YEAR / "robotpy" - self.pip_cache = self.cache_root / "pip_cache" - self.opkg_cache = self.cache_root / "opkg_cache" + self.set_cache_root(pathlib.Path.home() / "wpilib" / _WPILIB_YEAR / "robotpy") self._ssh: typing.Optional[SshController] = None self._cache_server: typing.Optional[CacheServer] = None @@ -84,6 +82,11 @@ def __init__(self, *, log_startup: bool = True): logger.info("RobotPy Installer %s", __version__) logger.info("-> caching files at %s", self.cache_root) + def set_cache_root(self, cache_root: pathlib.Path): + self.cache_root = cache_root + self.pip_cache = self.cache_root / "pip_cache" + self.opkg_cache = self.cache_root / "opkg_cache" + @contextlib.contextmanager def connect_to_robot( self, @@ -390,11 +393,12 @@ def _python_ipk_path(self) -> pathlib.Path: def is_python_downloaded(self) -> bool: return self._python_ipk_path.exists() - def download_python(self, use_certifi: bool): + def download_python(self, use_certifi: bool) -> pathlib.Path: self.opkg_cache.mkdir(parents=True, exist_ok=True) ipk_dst = self._python_ipk_path _urlretrieve(_PYTHON_IPK, ipk_dst, True, _make_ssl_context(use_certifi)) + return ipk_dst def install_python(self): """ @@ -443,6 +447,7 @@ def _extend_pip_args( no_deps: bool, pre: bool, requirements: typing.Iterable[str], + find_links: typing.Optional[pathlib.Path] = None, ): if pre: pip_args.append("--pre") @@ -452,6 +457,8 @@ def _extend_pip_args( pip_args.append("--ignore-installed") if no_deps: pip_args.append("--no-deps") + if find_links is not None: + pip_args.extend(["--find-links", str(find_links.absolute())]) for req in requirements: if cache: @@ -467,6 +474,8 @@ def pip_download( pre: bool, requirements: typing.Iterable[str], packages: typing.Iterable[str], + find_links: typing.Optional[pathlib.Path] = None, + no_index: bool = False, ): """ Specify Python package(s) to download, and store them in the cache. @@ -492,8 +501,6 @@ def pip_download( "--no-cache-dir", "--disable-pip-version-check", "download", - "--extra-index-url", - _ROBORIO_WHEELS, "--only-binary", ":all:", "--platform", @@ -508,6 +515,14 @@ def pip_download( str(self.pip_cache), ] + if no_index: + pip_args.append("--no-index") + else: + pip_args += [ + "--extra-index-url", + _ROBORIO_WHEELS, + ] + self._extend_pip_args( pip_args, None, @@ -516,6 +531,7 @@ def pip_download( no_deps, pre, requirements, + find_links=find_links, ) pip_args.extend(packages) diff --git a/setup.cfg b/setup.cfg index 5e3f055..b39e2ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,5 +41,6 @@ robotpy = deploy-info = robotpy_installer.cli_deploy_info:DeployInfo init = robotpy_installer.cli_init:Init installer = robotpy_installer.cli_installer:Installer + make-offline = robotpy_installer.cli_makeoffline:MakeOffline sync = robotpy_installer.cli_sync:Sync undeploy = robotpy_installer.cli_undeploy:Undeploy