[CLI] Migrate run (#5057)

### Changes

This PR adds two new commands to the `casp` CLI: `run local` and ` run
container`. Both of these commands run a one-off script against the
datastore, with the capability to use Clusterfuzz internal code. The
difference between them is **where** they run, as explained below.

#### Local
This version runs the script on the host machine, using the local code
of clusterfuzz. Basically, users need to have a cloned and configured
clusterfuzz and clusterfuzz-config repositories locally. The script they
want to run must be in the path `clusterfuzz/src/local/butler/scripts`,
just like the `butler.py` command.

#### Container
This version runs inside a Docker container. Choose an image using the
`-p` argument: `internal`, `external`, or `dev`. Then, provide the path
to your script, which will be mounted inside the container.

### How it works:

- `casp run container <script_name> -p <project>` runs the specified
script inside a Docker container (requires a project target: `dev`,
`internal`, or `external`).
- `casp run local <script_name> -c <config_dir>` runs the script locally
on the host machine (requires the path to the application config).
- Both commands default to a dry run; use `--non-dry-run` to enable
actual datastore writes.
- Use `--script_args <text>` to pass specific arguments to the script
being executed.

### Changes in butler

The only change in butler was adding a more descriptive print when there
are `script_args` in the `run` call. Here are screenshots from both
master and this branch showing that the behavior is the same:

<img width="2276" height="942" alt="image"
src="https://github.com/user-attachments/assets/40d4e873-52f8-4281-8d19-cfe5b2435c31"
/>
<img width="2276" height="942" alt="image"
src="https://github.com/user-attachments/assets/7718269e-0aae-4695-ba30-9d5ce6ac2d2e"
/>

### Demo prints

Here are some screenshots of `casp run` in action:

1. Running in local mode:
- `casp run local list_testcases_by_job --script_args centipede_v8_asan
-c ../clusterfuzz-config/configs/internal`
<img width="2276" height="942" alt="image"
src="https://github.com/user-attachments/assets/1b433277-f418-40d2-92a0-0ac7d41a61b9"
/>

2. Running the same script in container mode (note that the script was
placed in the root directory here):
- `casp run container list_testcases_by_job.py --script_args
centipede_v8_asan -p internal`
<img width="2276" height="942" alt="image"
src="https://github.com/user-attachments/assets/c50d9406-9041-4133-9daf-f084fe4e7c4d"
/>


Here's the script used in the demos for reproduction purposes:
```python
"""One-off script that lists all testcases of a given job"""

from clusterfuzz._internal.datastore import data_types
from clusterfuzz._internal.datastore import ndb_utils

def execute(args):
  """Executes the script"""
  try:
    query = ndb_utils.get_all_from_query(
      data_types.Testcase.query(data_types.Testcase.job_type == args.script_args[0]))

    print(f'Those are the testcase ids of job {args.script_args[0]}')
    for testcase in query:
      print(testcase.key.id())
  except Exception as e:
    print(e)
    
```

Author: PauloVLB

butler.py

@@ -98,6 +98,28 @@ def _setup_args_for_remote(parser):
   subparsers.add_parser('reboot', help='Reboot with `sudo reboot`.')
 
 
+def _add_run_subparser(toplevel_subparsers):
+  """Adds a parser for the `run` command."""
+  parser_run = toplevel_subparsers.add_parser(
+      'run', help='Run a one-off script against a datastore (e.g. migration).')
+  parser_run.add_argument(
+      'script_name',
+      help='The script module name under `./local/butler/scripts`.')
+  parser_run.add_argument(
+      '--script_args',
+      action='extend',
+      nargs='+',
+      help='Script specific arguments')
+  parser_run.add_argument(
+      '--non-dry-run',
+      action='store_true',
+      help='Run with actual datastore writes. Default to dry-run.')
+  parser_run.add_argument(
+      '-c', '--config-dir', required=True, help='Path to application config.')
+  parser_run.add_argument(
+      '--local', action='store_true', help='Run against local server instance.')
+
+
 def _add_package_subparser(toplevel_subparsers):
   """Adds a parser for the `package` command."""
   parser_package = toplevel_subparsers.add_parser(
@@ -391,25 +413,6 @@ def main():
   parser_run_server.add_argument(
       '--clean', action='store_true', help='Clear existing database data.')
 
-  parser_run = subparsers.add_parser(
-      'run', help='Run a one-off script against a datastore (e.g. migration).')
-  parser_run.add_argument(
-      'script_name',
-      help='The script module name under `./local/butler/scripts`.')
-  parser_run.add_argument(
-      '--script_args',
-      action='extend',
-      nargs='+',
-      help='Script specific arguments')
-  parser_run.add_argument(
-      '--non-dry-run',
-      action='store_true',
-      help='Run with actual datastore writes. Default to dry-run.')
-  parser_run.add_argument(
-      '-c', '--config-dir', required=True, help='Path to application config.')
-  parser_run.add_argument(
-      '--local', action='store_true', help='Run against local server instance.')
-
   parser_run_bot = subparsers.add_parser(
       'run_bot', help='Run a local clusterfuzz bot.')
   parser_run_bot.add_argument(
@@ -460,6 +463,7 @@ def main():
       default='us-central',
       help='Location for App Engine.')
 
+  _add_run_subparser(subparsers)
   _add_package_subparser(subparsers)
   _add_bootstrap_subparser(subparsers)
   _add_py_unittest_subparser(subparsers)

cli/casp/src/casp/commands/run.py

@@ -7,18 +7,163 @@
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is is "AS IS" BASIS,
+# distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Run command."""
 
+from pathlib import Path
+import subprocess
+import sys
+
+from casp.utils import config
+from casp.utils import container
+from casp.utils import docker_utils
+from casp.utils import local_butler
 import click
 
 
-@click.command(
+def common_options(func):
+  """Decorator to add common options to both 
+    local and container modes
+  """
+  func = click.option(
+      '--local',
+      is_flag=True,
+      default=False,
+      help='Run against local server instance.')(
+          func)
+  func = click.option(
+      '--non-dry-run',
+      is_flag=True,
+      default=False,
+      help=('Run with actual datastore writes. '
+            'Default to dry-run. The dry-run behavior needs to be implemented'
+            ' by the script itself, it is not enforced globally'))(
+                func)
+  func = click.option(
+      '--script_args', multiple=True, help='Script specific arguments')(
+          func)
+  func = click.argument('script_name')(func)
+  return func
+
+
+def _prepare_butler_run_args(non_dry_run: bool,
+                             local: bool,
+                             config_dir: str | None = None):
+  """Prepares common butler run arguments."""
+  butler_args = {}
+  if non_dry_run:
+    butler_args['non_dry_run'] = None
+  if local:
+    butler_args['local'] = None
+  if config_dir:
+    butler_args['config_dir'] = config_dir
+
+  return butler_args
+
+
+def _get_script_args_list(script_args: list[str]) -> list[str]:
+  """Gets the script args list and return it with the correct format.
+  (e.g) [arg1, arg2] -> --script_args=arg1 --script_args=arg2
+  """
+  script_args_list = []
+  for arg in script_args:
+    script_args_list.append(f'--script_args={arg}')
+  return script_args_list
+
+
+@click.group(
     name='run',
-    help='Run a one-off script against a datastore (e.g. migration).')
+    help=('Run a one-off script against a datastore (e.g. migration). '
+          'If running locally, the script must be in path '
+          'clusterfuzz/src/local/butler/scripts.'))
 def cli():
   """Run a one-off script against a datastore (e.g. migration)."""
-  click.echo('To be implemented...')
+
+
+@cli.command(name='local', help='Run the script locally (on the host machine).')
+@common_options
+@click.option(
+    '--config-dir',
+    '-c',
+    required=True,
+    type=click.Path(exists=True),
+    help='Path to application config.')
+def local_cmd(script_name: str, script_args: list[str] | None,
+              non_dry_run: bool, local: bool, config_dir: str):
+  """Run a one-off script locally."""
+  try:
+    butler_args = _prepare_butler_run_args(non_dry_run, local, config_dir)
+
+    command = local_butler.build_command('run', **butler_args)
+
+    command.append(script_name)
+    if script_args is not None:
+      script_args_list = _get_script_args_list(script_args)
+      command.extend(script_args_list)
+  except FileNotFoundError:
+    click.echo('butler.py not found in this directory.', err=True)
+    sys.exit(1)
+
+  try:
+    subprocess.run(command, check=True)
+  except FileNotFoundError:
+    click.echo('python not found in PATH.', err=True)
+    sys.exit(1)
+  except subprocess.CalledProcessError as e:
+    click.echo(f'Error running butler.py run: {e}', err=True)
+    sys.exit(1)
+
+
+@cli.command(
+    name='container',
+    help=('Run the script inside a Docker container. '
+          'The SCRIPT_NAME must be the path to it.'))
+@common_options
+@click.option(
+    '--project',
+    '-p',
+    help='The ClusterFuzz project to use.',
+    required=True,
+    type=click.Choice(
+        docker_utils.PROJECT_TO_IMAGE.keys(), case_sensitive=False),
+)
+def container_cmd(script_name: str, script_args: list[str] | None,
+                  non_dry_run: bool, local: bool, project: str):
+  """Run a one-off script inside a container."""
+  cfg = config.load_and_validate_config()
+
+  volumes, container_config_dir = docker_utils.prepare_docker_volumes(
+      cfg, str(container.CONTAINER_CONFIG_PATH / 'config'))
+
+  if not Path(script_name).exists():
+    click.echo(
+        f'Script {script_name} does not exist. Please provide the path to it',
+        err=True)
+    sys.exit(1)
+
+  host_script_path = Path(script_name).resolve()
+  container_script_path = (
+      container.CONTAINER_SCRIPTS_DIR / host_script_path.name)
+  docker_utils.add_volume(volumes, str(container_script_path),
+                          str(host_script_path))
+
+  butler_args = _prepare_butler_run_args(
+      non_dry_run, local, config_dir=str(container_config_dir))
+
+  subcommand = f'run {host_script_path.stem}'
+  if script_args:
+    script_args_list = _get_script_args_list(script_args)
+    subcommand += ' ' + ' '.join(script_args_list)
+
+  command = container.build_butler_command(subcommand, **butler_args)
+
+  if not docker_utils.run_command(
+      command,
+      volumes,
+      privileged=True,
+      image=docker_utils.PROJECT_TO_IMAGE[project],
+  ):
+    sys.exit(1)

cli/casp/src/casp/tests/commands/run_test.py

@@ -0,0 +1,144 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the run command.
+
+  For running all the tests, use (from the root of the project):
+  python -m unittest discover -s cli/casp/src/casp/tests -p run_test.py -v
+"""
+
+from pathlib import Path
+import unittest
+from unittest.mock import patch
+
+from casp.commands import run as run_command
+from click.testing import CliRunner
+
+
+class RunLocalCliTest(unittest.TestCase):
+  """Tests for the local run command."""
+
+  def setUp(self):
+    self.runner = CliRunner()
+    self.mock_local_butler = self.enterContext(
+        patch('casp.commands.run.local_butler', autospec=True))
+    self.mock_subprocess_run = self.enterContext(
+        patch('subprocess.run', autospec=True))
+
+  def test_run_local_success_basic(self):
+    """Tests successful execution of `casp run local` with minimal args."""
+    # mock build_command to return a list that we can inspect
+    self.mock_local_butler.build_command.return_value = [
+        'cmd', '--config-dir=test_config'
+    ]
+
+    with self.runner.isolated_filesystem():
+      Path('test_config').mkdir()
+      result = self.runner.invoke(
+          run_command.cli,
+          ['local', 'test_script', '--config-dir', 'test_config'])
+      self.assertEqual(0, result.exit_code, msg=result.output)
+
+      self.mock_local_butler.build_command.assert_called_once_with(
+          'run', config_dir='test_config')
+
+      expected_cmd = ['cmd', '--config-dir=test_config', 'test_script']
+      self.mock_subprocess_run.assert_called_once_with(expected_cmd, check=True)
+
+  def test_run_local_success_all_options(self):
+    """Tests `casp run local` with all options."""
+    # mock build_command behavior for flags
+    self.mock_local_butler.build_command.return_value = [
+        'cmd', '--non-dry-run', '--local', '--config-dir=test_config'
+    ]
+
+    with self.runner.isolated_filesystem():
+      Path('test_config').mkdir()
+      result = self.runner.invoke(run_command.cli, [
+          'local', 'test_script', '--config-dir', 'test_config',
+          '--non-dry-run', '--local', '--script_args', 'arg1', '--script_args',
+          'arg2'
+      ])
+      self.assertEqual(0, result.exit_code, msg=result.output)
+
+      self.mock_local_butler.build_command.assert_called_once_with(
+          'run', non_dry_run=None, local=None, config_dir='test_config')
+
+      expected_cmd = [
+          'cmd', '--non-dry-run', '--local', '--config-dir=test_config',
+          'test_script', '--script_args=arg1', '--script_args=arg2'
+      ]
+      self.mock_subprocess_run.assert_called_once_with(expected_cmd, check=True)
+
+
+class RunContainerCliTest(unittest.TestCase):
+  """Tests for the container run command."""
+
+  def setUp(self):
+    self.runner = CliRunner()
+    self.mock_config = self.enterContext(
+        patch('casp.commands.run.config', autospec=True))
+    self.mock_container = self.enterContext(
+        patch('casp.commands.run.container', autospec=True))
+    self.mock_docker_utils = self.enterContext(
+        patch('casp.commands.run.docker_utils', autospec=True))
+
+    # Mock config
+    self.mock_config.load_and_validate_config.return_value = {}
+
+    # Mock docker utils
+    self.mock_docker_utils.PROJECT_TO_IMAGE = {'dev': 'test-image'}
+    self.mock_docker_utils.prepare_docker_volumes.return_value = ({
+        'vol': 'bind'
+    }, Path('/container/config'))
+    self.mock_docker_utils.run_command.return_value = True
+
+    # Mock container
+    self.mock_container.CONTAINER_CONFIG_PATH = Path('/data/config')
+    self.mock_container.build_butler_command.return_value = [
+        'bash', '-c', 'cmd'
+    ]
+
+  def test_run_container_success(self):
+    """Tests successful execution of `casp run container`."""
+    with self.runner.isolated_filesystem():
+      Path('test_script').touch()
+      result = self.runner.invoke(run_command.cli, [
+          'container', 'test_script', '--project', 'dev', '--non-dry-run',
+          '--local', '--script_args', 'arg1', '--script_args', 'arg2'
+      ])
+      self.assertEqual(0, result.exit_code, msg=result.output)
+
+      self.mock_docker_utils.prepare_docker_volumes.assert_called_once()
+
+      # Expect None for flags (non_dry_run, local)
+      self.mock_container.build_butler_command.assert_called_once_with(
+          'run test_script --script_args=arg1 --script_args=arg2',
+          non_dry_run=None,
+          config_dir='/container/config',
+          local=None)
+      self.mock_docker_utils.run_command.assert_called_once_with(
+          ['bash', '-c', 'cmd'], {'vol': 'bind'},
+          privileged=True,
+          image='test-image')
+
+  def test_run_container_fail(self):
+    """Tests `casp run container` when docker run fails."""
+    self.mock_docker_utils.run_command.return_value = False
+    result = self.runner.invoke(
+        run_command.cli, ['container', 'test_script', '--project', 'dev'])
+    self.assertNotEqual(0, result.exit_code)
+
+
+if __name__ == '__main__':
+  unittest.main()

cli/casp/src/casp/utils/container.py

@@ -26,6 +26,9 @@
 # This allows the container to authenticate with Google Cloud services.
 CONTAINER_CREDENTIALS_PATH = Path('/root/.config/gcloud/')
 
+# The path to the directory containing butler scripts inside the container.
+CONTAINER_SCRIPTS_DIR = SRC_ROOT / 'local' / 'butler' / 'scripts'
+
 # The base command prefix for executing ClusterFuzz butler commands.
 # This ensures that commands are run with the correct Python environment
 # and logging settings within the container.