buildbot-effects: support flake references for remote repos

Avoids the painful workflow of cloning a repo into a tempdir just to
run a one-off effect. Commands now accept Nix flake reference syntax:

  buildbot-effects run github:org/repo/branch#my-effect
  buildbot-effects list github:org/repo/branch
  buildbot-effects list-schedules github:org/repo/branch
  buildbot-effects run-scheduled github:org/repo#schedule effect

When a flake ref is detected (by the presence of '#' in the positional
arg, or as an optional positional for list commands), we resolve it via
`nix flake metadata --json` to obtain the rev, branch, store path, and
locked URL — then pass the locked URL directly to builtins.getFlake
instead of constructing a git+file:// URL from a local checkout.

Closes: #584

Author: Mic92

buildbot_effects/buildbot_effects/__init__.py

@@ -96,11 +96,23 @@ def nix_command(*args: str) -> list[str]:
     return ["nix", "--extra-experimental-features", "nix-command flakes", *args]
 
 
+def _flake_url(opts: EffectsOptions, rev: str) -> str:
+    """Return the Nix flake URL to use with builtins.getFlake.
+
+    When a locked_url is available (from a resolved remote flake ref),
+    use it directly. Otherwise fall back to constructing a git+file:// URL
+    from the local path.
+    """
+    if opts.locked_url:
+        return opts.locked_url
+    return f"git+file://{opts.path}?rev={rev}#"
+
+
 def effect_function(opts: EffectsOptions) -> str:
     args = effects_args(opts)
     rev = args["rev"]
     escaped_args = json.dumps(json.dumps(args))
-    url = json.dumps(f"git+file://{opts.path}?rev={rev}#")
+    url = json.dumps(_flake_url(opts, rev))
     return f"""
       let
         flake = builtins.getFlake {url};
@@ -124,7 +136,7 @@ def scheduled_effect_function(opts: EffectsOptions) -> str:
     args = effects_args(opts)
     rev = args["rev"]
     escaped_args = json.dumps(json.dumps(args))
-    url = json.dumps(f"git+file://{opts.path}?rev={rev}#")
+    url = json.dumps(_flake_url(opts, rev))
     return f"""
       let
         flake = builtins.getFlake {url};

buildbot_effects/buildbot_effects/cli.py

@@ -2,8 +2,11 @@
 
 import argparse
 import json
+import shlex
+import subprocess
 import sys
 from pathlib import Path
+from typing import Any
 
 from . import (
     instantiate_effects,
@@ -16,12 +19,53 @@
 from .options import EffectsOptions
 
 
+def resolve_flake(flake_ref: str, *, debug: bool = False) -> dict[str, Any]:
+    """Run `nix flake metadata --json` and return the parsed JSON."""
+    cmd = [
+        "nix",
+        "--extra-experimental-features",
+        "nix-command flakes",
+        "flake",
+        "metadata",
+        "--json",
+        flake_ref,
+    ]
+    if debug:
+        print("$", shlex.join(cmd), file=sys.stderr)
+    proc = subprocess.run(cmd, check=True, text=True, capture_output=True)
+    return json.loads(proc.stdout)
+
+
+def options_from_flake_ref(flake_ref: str, base: EffectsOptions) -> EffectsOptions:
+    """Resolve a flake reference to EffectsOptions via nix flake metadata."""
+    meta = resolve_flake(flake_ref, debug=base.debug)
+    locked = meta.get("locked", {})
+    return EffectsOptions(
+        secrets=base.secrets,
+        path=Path(meta.get("path", "")),
+        repo="",
+        rev=locked.get("rev"),
+        branch=locked.get("ref"),
+        url=meta.get("resolvedUrl", meta.get("url", "")),
+        locked_url=meta.get("lockedUrl", ""),
+        debug=base.debug,
+    )
+
+
 def list_command(_args: argparse.Namespace, options: EffectsOptions) -> None:
+    if _args.flake_ref:
+        options = options_from_flake_ref(_args.flake_ref, options)
     json.dump(list_effects(options), fp=sys.stdout, indent=2)
 
 
 def run_command(args: argparse.Namespace, options: EffectsOptions) -> None:
     effect = args.effect
+
+    # Support flakeref#effect syntax: github:org/repo/branch#my-effect
+    if "#" in effect:
+        flake_ref, _, effect = effect.partition("#")
+        options = options_from_flake_ref(flake_ref, options)
+
     drv_path = instantiate_effects(effect, options)
     if drv_path == "":
         print(f"Effect {effect} not found or not runnable for {options}")
@@ -41,17 +85,26 @@ def run_all_command(_args: argparse.Namespace, _options: EffectsOptions) -> None
 
 def list_schedules_command(_args: argparse.Namespace, options: EffectsOptions) -> None:
     """List all scheduled effects defined in the flake."""
+    if _args.flake_ref:
+        options = options_from_flake_ref(_args.flake_ref, options)
     json.dump(list_scheduled_effects(options), fp=sys.stdout, indent=2)
 
 
 def run_scheduled_command(args: argparse.Namespace, options: EffectsOptions) -> None:
     """Run a specific effect from a schedule."""
     schedule_name = args.schedule_name
     effect = args.effect
+
+    # Support flakeref#schedule syntax: github:org/repo/branch#my-schedule
+    if "#" in schedule_name:
+        flake_ref, _, schedule_name = schedule_name.partition("#")
+        options = options_from_flake_ref(flake_ref, options)
+
     drv_path = instantiate_scheduled_effect(schedule_name, effect, options)
     if drv_path == "":
         print(
-            f"Scheduled effect {schedule_name}/{effect} not found or not runnable for {options}"
+            f"Scheduled effect {schedule_name}/{effect} not found or not runnable"
+            f" for {options}"
         )
         return
     drvs = parse_derivation(drv_path)
@@ -64,7 +117,18 @@ def run_scheduled_command(args: argparse.Namespace, options: EffectsOptions) ->
 
 
 def parse_args() -> tuple[argparse.Namespace, EffectsOptions]:
-    parser = argparse.ArgumentParser(description="Run effects from a hercules-ci flake")
+    parser = argparse.ArgumentParser(
+        description="Run effects from a hercules-ci flake",
+        epilog=(
+            "Flake reference syntax:\n"
+            "  Commands accept flake references to operate on remote repositories\n"
+            "  without requiring a local checkout:\n\n"
+            "  buildbot-effects run github:org/repo/branch#my-effect\n"
+            "  buildbot-effects list github:org/repo/branch\n"
+            "  buildbot-effects run-scheduled github:org/repo#schedule effect\n"
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
     parser.add_argument(
         "--secrets",
         type=Path,
@@ -104,17 +168,23 @@ def parse_args() -> tuple[argparse.Namespace, EffectsOptions]:
     )
     list_parser = subparser.add_parser(
         "list",
-        help="List available effects",
+        help="List available effects (optionally from a flake reference)",
     )
     list_parser.set_defaults(command=list_command)
+    list_parser.add_argument(
+        "flake_ref",
+        nargs="?",
+        help="Flake reference (e.g. github:org/repo/branch)",
+    )
+
     run_parser = subparser.add_parser(
         "run",
-        help="Run an effect",
+        help="Run an effect (supports flakeref#effect syntax)",
     )
     run_parser.set_defaults(command=run_command)
     run_parser.add_argument(
         "effect",
-        help="Effect to run",
+        help="Effect to run, or flakeref#effect (e.g. github:org/repo/branch#deploy)",
     )
     run_all_parser = subparser.add_parser(
         "run-all",
@@ -124,9 +194,14 @@ def parse_args() -> tuple[argparse.Namespace, EffectsOptions]:
 
     list_schedules_parser = subparser.add_parser(
         "list-schedules",
-        help="List all scheduled effects defined in the flake",
+        help="List all scheduled effects (optionally from a flake reference)",
     )
     list_schedules_parser.set_defaults(command=list_schedules_command)
+    list_schedules_parser.add_argument(
+        "flake_ref",
+        nargs="?",
+        help="Flake reference (e.g. github:org/repo/branch)",
+    )
 
     run_scheduled_parser = subparser.add_parser(
         "run-scheduled",
@@ -135,7 +210,7 @@ def parse_args() -> tuple[argparse.Namespace, EffectsOptions]:
     run_scheduled_parser.set_defaults(command=run_scheduled_command)
     run_scheduled_parser.add_argument(
         "schedule_name",
-        help="Name of the schedule (from onSchedule.<name>)",
+        help="Schedule name, or flakeref#schedule (e.g. github:org/repo#my-schedule)",
     )
     run_scheduled_parser.add_argument(
         "effect",

buildbot_effects/buildbot_effects/options.py

@@ -13,4 +13,5 @@ class EffectsOptions:
     branch: str | None = None
     url: str | None = None
     tag: str | None = None
+    locked_url: str | None = None
     debug: bool = False

buildbot_effects/tests/test_cli.py

@@ -0,0 +1,63 @@
+"""Tests for CLI flake ref support."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from buildbot_effects.cli import run_command
+from buildbot_effects.options import EffectsOptions
+
+FAKE_METADATA: dict = {
+    "url": "github:my-org/my-repo/main",
+    "resolvedUrl": "github:my-org/my-repo/abc1234",
+    "lockedUrl": "github:my-org/my-repo/abc1234def5678abc1234def5678abc1234def567",
+    "path": "/nix/store/abc123-source",
+    "locked": {
+        "rev": "abc1234def5678abc1234def5678abc1234def567",
+        "ref": "main",
+        "lastModified": 1700000000,
+    },
+}
+
+
+def _base_options() -> EffectsOptions:
+    return EffectsOptions(
+        secrets=Path("/tmp/secrets.json"),  # noqa: S108
+        debug=True,
+    )
+
+
+class TestRunCommandFlakeRef:
+    def test_flake_ref_resolves_and_splits_effect(self) -> None:
+        args = MagicMock()
+        args.effect = "github:my-org/my-repo/main#deploy"
+
+        with (
+            patch(
+                "buildbot_effects.cli.resolve_flake",
+                return_value=FAKE_METADATA,
+            ) as mock_resolve,
+            patch(
+                "buildbot_effects.cli.instantiate_effects", return_value=""
+            ) as mock_inst,
+        ):
+            run_command(args, _base_options())
+
+        mock_resolve.assert_called_once_with("github:my-org/my-repo/main", debug=True)
+        assert mock_inst.call_args[0][0] == "deploy"
+
+    def test_plain_effect_skips_resolution(self) -> None:
+        args = MagicMock()
+        args.effect = "deploy"
+
+        with (
+            patch("buildbot_effects.cli.resolve_flake") as mock_resolve,
+            patch(
+                "buildbot_effects.cli.instantiate_effects", return_value=""
+            ) as mock_inst,
+        ):
+            run_command(args, _base_options())
+
+        mock_resolve.assert_not_called()
+        assert mock_inst.call_args[0][0] == "deploy"

buildbot_effects/tests/test_flake_url.py

@@ -0,0 +1,29 @@
+"""Test _flake_url: the bridge between EffectsOptions and builtins.getFlake."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from buildbot_effects import _flake_url
+from buildbot_effects.options import EffectsOptions
+
+
+class TestFlakeUrl:
+    def test_local_path_fallback(self) -> None:
+        opts = EffectsOptions(path=Path("/home/user/my-repo"))
+        assert (
+            _flake_url(opts, "abc1234") == "git+file:///home/user/my-repo?rev=abc1234#"
+        )
+
+    def test_locked_url_used(self) -> None:
+        opts = EffectsOptions(
+            path=Path("/nix/store/xyz-source"),
+            locked_url="github:org/repo/abc1234def5678",
+        )
+        assert _flake_url(opts, "abc1234") == "github:org/repo/abc1234def5678"
+
+    def test_empty_locked_url_falls_back(self) -> None:
+        opts = EffectsOptions(path=Path("/home/user/my-repo"), locked_url="")
+        assert (
+            _flake_url(opts, "abc1234") == "git+file:///home/user/my-repo?rev=abc1234#"
+        )