Merge pull request #176 from davidbrochart/py_multi_package

Handle multiple Python packages

Author: Steven Silvester

docs/source/get_started/making_first_release.md

@@ -23,6 +23,13 @@ already uses Jupyter Releaser.
     owner2/repo2,token2
     ```
 
+    If you have multiple Python packages in one repository, you can point to them as follows:
+
+    ```text
+    owner1/repo1/path/to/package1,token1
+    owner1/repo1/path/to/package2,token1
+    ```
+
 - If the repo generates npm release(s), add access token for [npm](https://docs.npmjs.com/creating-and-viewing-access-tokens), saved as `NPM_TOKEN` in "Secrets".
 
 ## Draft Changelog

docs/source/how_to_guides/convert_repo.md

@@ -23,11 +23,18 @@ A. Prep the `jupyter_releaser` fork:
       _Note_ For security reasons, it is recommended that you scope the access
       to a single repository, and use a variable called `PYPI_TOKEN_MAP` that is formatted as follows:
 
-      ```
+      ```text
       owner1/repo1,token1
       owner2/repo2,token2
       ```
 
+      If you have multiple Python packages in one repository, you can point to them as follows:
+
+      ```text
+      owner1/repo1/path/to/package1,token1
+      owner1/repo1/path/to/package2,token1
+      ```
+
 - [ ] If needed, add access token for [npm](https://docs.npmjs.com/creating-and-viewing-access-tokens), saved as `NPM_TOKEN`.
 
 B. Prep target repository:

jupyter_releaser/cli.py

@@ -171,6 +171,16 @@ def main(force):
     )
 ]
 
+python_packages_options = [
+    click.option(
+        "--python-packages",
+        envvar="RH_PYTHON_PACKAGES",
+        default=["."],
+        multiple=True,
+        help="The list of paths to Python packages",
+    )
+]
+
 dry_run_options = [
     click.option(
         "--dry-run", is_flag=True, envvar="RH_DRY_RUN", help="Run as a dry run"
@@ -273,10 +283,15 @@ def prep_git(ref, branch, repo, auth, username, git_url):
 @main.command()
 @add_options(version_spec_options)
 @add_options(version_cmd_options)
+@add_options(python_packages_options)
 @use_checkout_dir()
-def bump_version(version_spec, version_cmd):
+def bump_version(version_spec, version_cmd, python_packages):
     """Prep git and env variables and bump version"""
-    lib.bump_version(version_spec, version_cmd)
+    prev_dir = os.getcwd()
+    for python_package in python_packages:
+        os.chdir(python_package)
+        lib.bump_version(version_spec, version_cmd)
+        os.chdir(prev_dir)
 
 
 @main.command()
@@ -363,13 +378,24 @@ def check_changelog(
 
 @main.command()
 @add_options(dist_dir_options)
+@add_options(python_packages_options)
 @use_checkout_dir()
-def build_python(dist_dir):
+def build_python(dist_dir, python_packages):
     """Build Python dist files"""
-    if not util.PYPROJECT.exists() and not util.SETUP_PY.exists():
-        util.log("Skipping build-python since there are no python package files")
-        return
-    python.build_dist(dist_dir)
+    prev_dir = os.getcwd()
+    clean = True
+    for python_package in python_packages:
+        os.chdir(python_package)
+        if not util.PYPROJECT.exists() and not util.SETUP_PY.exists():
+            util.log(
+                f"Skipping build-python in {python_package} since there are no python package files"
+            )
+        else:
+            python.build_dist(
+                Path(os.path.relpath(".", python_package)) / dist_dir, clean=clean
+            )
+            clean = False
+        os.chdir(prev_dir)
 
 
 @main.command()
@@ -580,6 +606,7 @@ def extract_release(auth, dist_dir, dry_run, release_url, npm_install_options):
     default="https://pypi.org/simple/",
 )
 @add_options(dry_run_options)
+@add_options(python_packages_options)
 @click.argument("release-url", nargs=1, required=False)
 @use_checkout_dir()
 def publish_assets(
@@ -591,18 +618,21 @@ def publish_assets(
     twine_registry,
     dry_run,
     release_url,
+    python_packages,
 ):
     """Publish release asset(s)"""
-    lib.publish_assets(
-        dist_dir,
-        npm_token,
-        npm_cmd,
-        twine_cmd,
-        npm_registry,
-        twine_registry,
-        dry_run,
-        release_url,
-    )
+    for python_package in python_packages:
+        lib.publish_assets(
+            dist_dir,
+            npm_token,
+            npm_cmd,
+            twine_cmd,
+            npm_registry,
+            twine_registry,
+            dry_run,
+            release_url,
+            python_package,
+        )
 
 
 @main.command()

jupyter_releaser/lib.py

@@ -381,6 +381,7 @@ def publish_assets(
     twine_registry,
     dry_run,
     release_url,
+    python_package,
 ):
     """Publish release asset(s)"""
     os.environ["NPM_REGISTRY"] = npm_registry
@@ -393,7 +394,7 @@ def publish_assets(
             util.run("npm whoami")
 
     if len(glob(f"{dist_dir}/*.whl")):
-        twine_token = python.get_pypi_token(release_url)
+        twine_token = python.get_pypi_token(release_url, python_package)
 
     if dry_run:
         # Start local pypi server with no auth, allowing overwrites,

jupyter_releaser/python.py

@@ -17,16 +17,17 @@
 SETUP_PY = util.SETUP_PY
 
 
-def build_dist(dist_dir):
+def build_dist(dist_dir, clean=True):
     """Build the python dist files into a dist folder"""
     # Clean the dist folder of existing npm tarballs
     os.makedirs(dist_dir, exist_ok=True)
     dest = Path(dist_dir)
-    for pkg in glob(f"{dest}/*.gz") + glob(f"{dest}/*.whl"):
-        os.remove(pkg)
+    if clean:
+        for pkg in glob(f"{dest}/*.gz") + glob(f"{dest}/*.whl"):
+            os.remove(pkg)
 
     if PYPROJECT.exists():
-        util.run(f"python -m build --outdir {dest} .", quiet=True)
+        util.run(f"python -m build --outdir {dest} .", quiet=True, show_cwd=True)
     elif SETUP_PY.exists():
         util.run(f"python setup.py sdist --dist-dir {dest}", quiet=True)
         util.run(f"python setup.py bdist_wheel --dist-dir {dest}", quiet=True)
@@ -60,7 +61,7 @@ def check_dist(dist_file, test_cmd=""):
         util.run(f"{bin_path}/{test_cmd}")
 
 
-def get_pypi_token(release_url):
+def get_pypi_token(release_url, python_package):
     """Get the PyPI token
 
     Note: Do not print the token in CI since it will not be sanitized
@@ -70,6 +71,8 @@ def get_pypi_token(release_url):
     if pypi_token_map and release_url:
         parts = release_url.replace("https://github.com/", "").split("/")
         repo_name = f"{parts[0]}/{parts[1]}"
+        if python_package != ".":
+            repo_name += f"/{python_package}"
         util.log(f"Looking for PYPI token for {repo_name} in token map")
         for line in pypi_token_map.splitlines():
             name, _, token = line.partition(",")

jupyter_releaser/tee.py

@@ -140,10 +140,11 @@ def tee_func(line: bytes, sink: List[str], pipe: Optional[Any]) -> None:
 
 
 def run(args: Union[str, List[str]], **kwargs: Any) -> CompletedProcess:
-    """Drop-in replacement for subprocerss.run that behaves like tee.
+    """Drop-in replacement for subprocess.run that behaves like tee.
     Extra arguments added by our version:
     echo: False - Prints command before executing it.
     quiet: False - Avoid printing output
+    show_cwd: False - Prints the current working directory.
     """
     if isinstance(args, str):
         cmd = args
@@ -158,7 +159,11 @@ def run(args: Union[str, List[str]], **kwargs: Any) -> CompletedProcess:
     if kwargs.get("echo", False):
         # This is modified from the default implementation since
         # we want all output to be interleved on the same stream
-        print(f"COMMAND: {cmd}", file=sys.stderr)
+        prefix = "COMMAND"
+        if kwargs.pop("show_cwd", False):
+            prefix += f" (in '{os.getcwd()}')"
+        prefix += ":"
+        print(f"{prefix} {cmd}", file=sys.stderr)
 
     loop = asyncio.get_event_loop()
     result = loop.run_until_complete(_stream_subprocess(cmd, **kwargs))

jupyter_releaser/tests/conftest.py

@@ -75,6 +75,11 @@ def py_package(git_repo):
     return testutil.create_python_package(git_repo)
 
 
+@fixture
+def py_multipackage(git_repo):
+    return testutil.create_python_package(git_repo, multi=True)
+
+
 @fixture
 def npm_package(git_repo):
     return testutil.create_npm_package(git_repo)

jupyter_releaser/tests/test_cli.py

@@ -162,6 +162,7 @@ def test_list_envvars(runner):
 output: RH_CHANGELOG_OUTPUT
 post-version-message: RH_POST_VERSION_MESSAGE
 post-version-spec: RH_POST_VERSION_SPEC
+python-packages: RH_PYTHON_PACKAGES
 ref: RH_REF
 release-message: RH_RELEASE_MESSAGE
 repo: RH_REPOSITORY
@@ -596,6 +597,64 @@ def helper(path, **kwargs):
     assert "after-extract-release" in log
 
 
+@pytest.mark.skipif(
+    os.name == "nt" and sys.version_info.major == 3 and sys.version_info.minor < 8,
+    reason="See https://bugs.python.org/issue26660",
+)
+def test_extract_dist_multipy(
+    py_multipackage, runner, mocker, open_mock, tmp_path, git_prep
+):
+    git_repo = py_multipackage[0]["abs_path"]
+    changelog_entry = mock_changelog_entry(git_repo, runner, mocker)
+
+    # Create the dist files
+    dist_dir = normalize_path(Path(util.CHECKOUT_NAME).resolve() / "dist")
+    for package in py_multipackage:
+        run(
+            f"python -m build . -o {dist_dir}",
+            cwd=Path(util.CHECKOUT_NAME) / package["rel_path"],
+        )
+
+    # Finalize the release
+    runner(["tag-release"])
+
+    os.makedirs("staging")
+    shutil.move(f"{util.CHECKOUT_NAME}/dist", "staging")
+
+    def helper(path, **kwargs):
+        return MockRequestResponse(f"{git_repo}/staging/dist/{path}")
+
+    get_mock = mocker.patch("requests.get", side_effect=helper)
+
+    tag_name = f"v{VERSION_SPEC}"
+
+    dist_names = [osp.basename(f) for f in glob("staging/dist/*.*")]
+    releases = [
+        dict(
+            tag_name=tag_name,
+            target_commitish=util.get_branch(),
+            assets=[dict(name=dist_name, url=dist_name) for dist_name in dist_names],
+        )
+    ]
+    sha = run("git rev-parse HEAD", cwd=util.CHECKOUT_NAME)
+
+    tags = [dict(ref=f"refs/tags/{tag_name}", object=dict(sha=sha))]
+    url = normalize_path(osp.join(os.getcwd(), util.CHECKOUT_NAME))
+    open_mock.side_effect = [
+        MockHTTPResponse(releases),
+        MockHTTPResponse(tags),
+        MockHTTPResponse(dict(html_url=url)),
+    ]
+
+    runner(["extract-release", HTML_URL])
+    assert len(open_mock.mock_calls) == 2
+    assert len(get_mock.mock_calls) == len(dist_names) == 2 * len(py_multipackage)
+
+    log = get_log()
+    assert "before-extract-release" not in log
+    assert "after-extract-release" in log
+
+
 @pytest.mark.skipif(
     os.name == "nt" and sys.version_info.major == 3 and sys.version_info.minor < 8,
     reason="See https://bugs.python.org/issue26660",

jupyter_releaser/tests/test_functions.py

@@ -1,6 +1,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import json
+import os
 import shutil
 from pathlib import Path
 
@@ -30,6 +31,16 @@ def test_get_version_python(py_package):
     assert util.get_version() == "0.0.2a0"
 
 
+def test_get_version_multipython(py_multipackage):
+    prev_dir = os.getcwd()
+    for package in py_multipackage:
+        os.chdir(package["rel_path"])
+        assert util.get_version() == "0.0.1"
+        util.bump_version("0.0.2a0")
+        assert util.get_version() == "0.0.2a0"
+        os.chdir(prev_dir)
+
+
 def test_get_version_npm(npm_package):
     assert util.get_version() == "1.0.0"
     npm = util.normalize_path(shutil.which("npm"))