From d8ff0d661bc25763e607a9171e8762af13a4a1f7 Mon Sep 17 00:00:00 2001 From: elazar Date: Wed, 4 Feb 2026 15:39:03 -0600 Subject: [PATCH] Add @podmanssh connector for remote Podman container operations Extends DockerSSHConnector with extensible docker_cmd class variable to support podman via minimal PodmanSSHConnector subclass. Eliminates code duplication and follows established @docker/@podman connector pattern. Features: - Add docker_cmd class variable to DockerSSHConnector (defaults to 'docker') - Replace hardcoded 'docker' strings with self.docker_cmd in all commands - Create PodmanSSHConnector subclass with docker_cmd = 'podman' override - Add podmanssh entry point to pyproject.toml - Implement comprehensive test coverage for both connector variants - Enhanced documentation with usage examples and feature descriptions Usage: pyinfra @podmanssh/remotehost:alpine:3.8 operations/deploy.py pyinfra @podmanssh/web1:nginx:latest,@podmanssh/web2:nginx:latest deploy.py Co-Authored-By: Claude Opus 4.5 --- docs/connectors.rst | 2 +- pyproject.toml | 1 + src/pyinfra/connectors/docker.py | 8 + src/pyinfra/connectors/dockerssh.py | 99 +++++++++- tests/test_connectors/test_dockerssh.py | 240 +++++++++++++++++++++++- 5 files changed, 338 insertions(+), 12 deletions(-) diff --git a/docs/connectors.rst b/docs/connectors.rst index 9f3a51cc1..57a5ec4eb 100644 --- a/docs/connectors.rst +++ b/docs/connectors.rst @@ -13,7 +13,7 @@ Connectors enable pyinfra to integrate with other tools out of the box. Connecto + Implement how commands are executed (``@ssh``, ``@local``) + Generate inventory hosts and data (``@terraform`` and ``@vagrant``) -+ Both of the above (``@docker``) ++ Both of the above (``@docker``, ``@podman``, and other container connectors) Each connector page is listed below and contains examples as well as a list of available data that can be used to configure the connector. diff --git a/pyproject.toml b/pyproject.toml index 708b269ef..3e8aab3b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ podman = "pyinfra.connectors.docker:PodmanConnector" local = "pyinfra.connectors.local:LocalConnector" ssh = "pyinfra.connectors.ssh:SSHConnector" dockerssh = "pyinfra.connectors.dockerssh:DockerSSHConnector" +podmanssh = "pyinfra.connectors.dockerssh:PodmanSSHConnector" # Inventory only connectors terraform = "pyinfra.connectors.terraform:TerraformInventoryConnector" vagrant = "pyinfra.connectors.vagrant:VagrantInventoryConnector" diff --git a/src/pyinfra/connectors/docker.py b/src/pyinfra/connectors/docker.py index 556459dda..66b65acaf 100644 --- a/src/pyinfra/connectors/docker.py +++ b/src/pyinfra/connectors/docker.py @@ -68,6 +68,10 @@ class DockerConnector(BaseConnector): The Docker connector is great for testing pyinfra operations locally, rather than connecting to a remote host over SSH each time. This gives you a fast, local-first devloop to iterate on when writing deploys, operations or facts. + + .. note:: + + For running Docker containers on remote hosts, see the :doc:`dockerssh` connector. """ # enable the use of other docker cli compatible tools like podman @@ -364,6 +368,10 @@ class PodmanConnector(DockerConnector): The Podman connector is great for testing pyinfra operations locally, rather than connecting to a remote host over SSH each time. This gives you a fast, local-first devloop to iterate on when writing deploys, operations or facts. + + .. note:: + + For running Podman containers on remote hosts, see the :doc:`podmanssh` connector. """ docker_cmd = "podman" diff --git a/src/pyinfra/connectors/dockerssh.py b/src/pyinfra/connectors/dockerssh.py index e7dbd3872..d10913ce7 100644 --- a/src/pyinfra/connectors/dockerssh.py +++ b/src/pyinfra/connectors/dockerssh.py @@ -33,6 +33,10 @@ class DockerSSHConnector(BaseConnector): The ``@dockerssh`` connector allows you to run commands on Docker containers \ on a remote machine. + .. note:: + + For running Podman containers on remote hosts, see the :doc:`podmanssh` connector. + .. code:: shell # A Docker base image must be provided @@ -43,6 +47,8 @@ class DockerSSHConnector(BaseConnector): """ handles_execution = True + # enable the use of other docker cli compatible tools like podman + docker_cmd = "docker" ssh: SSHConnector @@ -77,11 +83,11 @@ def connect(self) -> None: return try: - with progress_spinner({"docker run"}): + with progress_spinner({f"{self.docker_cmd} run"}): # last line is the container ID status, output = self.ssh.run_shell_command( StringCommand( - "docker", + self.docker_cmd, "run", "-d", self.host.data.docker_image, @@ -103,20 +109,23 @@ def connect(self) -> None: def disconnect(self) -> None: container_id = self.host.host_data["docker_container_id"][:12] - with progress_spinner({"docker commit"}): - _, output = self.ssh.run_shell_command(StringCommand("docker", "commit", container_id)) + with progress_spinner({f"{self.docker_cmd} commit"}): + _, output = self.ssh.run_shell_command( + StringCommand(self.docker_cmd, "commit", container_id) + ) # Last line is the image ID, get sha256:[XXXXXXXXXX]... image_id = output.stdout_lines[-1][7:19] - with progress_spinner({"docker rm"}): + with progress_spinner({f"{self.docker_cmd} rm"}): self.ssh.run_shell_command( - StringCommand("docker", "rm", "-f", container_id), + StringCommand(self.docker_cmd, "rm", "-f", container_id), ) logger.info( - "{0}docker build complete, image ID: {1}".format( + "{0}{1} build complete, image ID: {2}".format( self.host.print_prefix, + self.docker_cmd, click.style(image_id, bold=True), ), ) @@ -138,7 +147,7 @@ def run_shell_command( docker_flags = "-it" if local_arguments.get("_get_pty") else "-i" docker_command = StringCommand( - "docker", + self.docker_cmd, "exec", docker_flags, container_id, @@ -192,7 +201,7 @@ def put_file( try: docker_id = self.host.host_data["docker_container_id"] docker_command = StringCommand( - "docker", + self.docker_cmd, "cp", remote_temp_filename, f"{docker_id}:{remote_filename}", @@ -246,7 +255,7 @@ def get_file( try: docker_id = self.host.host_data["docker_container_id"] docker_command = StringCommand( - "docker", + self.docker_cmd, "cp", f"{docker_id}:{remote_filename}", remote_temp_filename, @@ -295,3 +304,73 @@ def remote_remove(self, filename, print_output: bool = False, print_input: bool if not remove_status: raise IOError(output.stderr) + + +@memoize +def show_warning_podman() -> None: + logger.warning("The @podmanssh connector is in beta!") + + +class PodmanSSHConnector(DockerSSHConnector): + """ + **Note**: this connector is in beta! + + The ``@podmanssh`` connector allows you to run commands on Podman containers + on a remote machine over SSH. This is useful when you need to manage containers + running on remote hosts where Podman is installed instead of Docker. + + .. note:: + + This connector requires SSH access to the remote host and Podman to be installed + on the target machine. It operates similarly to ``@dockerssh`` but uses the + ``podman`` command instead of ``docker``. + + + **Remote container creation**: Creates a new container from the specified image on the remote host + + **Command execution**: Runs operations inside the container via ``podman exec`` + + **File operations**: Supports uploading/downloading files to/from containers via ``podman cp`` + + **Container cleanup**: Automatically commits and removes containers when operations complete + + .. code:: shell + + # A Podman base image must be provided + pyinfra @podmanssh/remotehost:alpine:3.8 ... + + # Run operations on multiple remote Podman containers in parallel + pyinfra @podmanssh/web1:nginx:latest,@podmanssh/web2:nginx:latest deploy.py + + # Use with specific SSH connection settings + pyinfra @podmanssh/production-server:alpine:3.18 --sudo --port 2222 operations/ + + **Comparison with other connectors:** + + + Use ``@podman`` for local Podman containers + + Use ``@dockerssh`` for remote Docker containers + + Use ``@podmanssh`` for remote Podman containers (this connector) + + The Podman SSH connector is particularly useful in environments where: + + + Docker is not available but Podman is installed + + Rootless containers are preferred for security + + You need OCI-compliant container operations on remote hosts + """ + + docker_cmd = "podman" + + @override + @staticmethod + def make_names_data(name): + try: + hostname, image = name.split(":", 1) + except (AttributeError, ValueError): # failure to parse the name + raise InventoryError("No ssh host or podman base image provided!") + + if not image: + raise InventoryError("No podman base image provided!") + + show_warning_podman() + + yield ( + "@podmanssh/{0}:{1}".format(hostname, image), + {"ssh_hostname": hostname, "docker_image": image}, + ["@podmanssh"], + ) diff --git a/tests/test_connectors/test_dockerssh.py b/tests/test_connectors/test_dockerssh.py index 5948e4a92..4e93ec22d 100644 --- a/tests/test_connectors/test_dockerssh.py +++ b/tests/test_connectors/test_dockerssh.py @@ -36,7 +36,7 @@ def fake_ssh_docker_shell( # This is a bit messy. But it's easier than trying to swap out a mock # when it needs to be used... - if fake_ssh_docker_shell.custom_command: + if hasattr(fake_ssh_docker_shell, "custom_command") and fake_ssh_docker_shell.custom_command: custom_command, status, output = fake_ssh_docker_shell.custom_command if str(command) == custom_command: fake_ssh_docker_shell.ran_custom_command = True @@ -45,6 +45,39 @@ def fake_ssh_docker_shell( raise PyinfraError("Invalid Command: {0}".format(command)) +def fake_ssh_podman_shell( + self, + command, + print_output=False, + print_input=False, + **command_kwargs, +): + if str(command) == "podman run -d not-an-image tail -f /dev/null": + return (True, CommandOutput([OutputLine("stdout", "containerid")])) + + if str(command) == "podman commit containerid": + return (True, CommandOutput([OutputLine("stdout", "sha256:blahsomerandomstringdata")])) + + if str(command) == "podman rm -f containerid": + return (True, CommandOutput([])) + + if str(command).startswith("rm -f"): + return (True, CommandOutput([])) + + if "$TMPDIR" in str(command): + return (True, CommandOutput([])) + + # This is a bit messy. But it's easier than trying to swap out a mock + # when it needs to be used... + if hasattr(fake_ssh_podman_shell, "custom_command") and fake_ssh_podman_shell.custom_command: + custom_command, status, output = fake_ssh_podman_shell.custom_command + if str(command) == custom_command: + fake_ssh_podman_shell.ran_custom_command = True + return (status, output) + + raise PyinfraError("Invalid Command: {0}".format(command)) + + def get_docker_command(command): shell_command = make_unix_command(command).get_raw_value() shell_command = shlex.quote(shell_command) @@ -52,6 +85,13 @@ def get_docker_command(command): return docker_command +def get_podman_command(command): + shell_command = make_unix_command(command).get_raw_value() + shell_command = shlex.quote(shell_command) + podman_command = "podman exec -it containerid sh -c {0}".format(shell_command) + return podman_command + + @patch("pyinfra.connectors.ssh.SSHConnector.connect", MagicMock()) @patch("pyinfra.connectors.ssh.SSHConnector.run_shell_command", fake_ssh_docker_shell) @patch("pyinfra.api.util.open", mock_open(read_data="test!"), create=True) @@ -248,3 +288,201 @@ def test_get_file_error(self, fake_get_file): host.get_file("not-a-file", "not-another-file", print_output=True) assert str(ex.exception) == "failed to copy file over ssh" + + +@patch("pyinfra.connectors.ssh.SSHConnector.connect", MagicMock()) +@patch("pyinfra.connectors.ssh.SSHConnector.run_shell_command", fake_ssh_podman_shell) +@patch("pyinfra.api.util.open", mock_open(read_data="test!"), create=True) +class TestPodmanSSHConnector(TestCase): + def setUp(self): + # reset custom command for shell + fake_ssh_podman_shell.custom_command = None + fake_ssh_podman_shell.ran_custom_command = False + + def test_connect_host(self): + inventory = make_inventory(("somehost", "anotherhost")) + state = State(inventory, Config()) + host = inventory.get_host("somehost") + host.connect(reason=True) + assert len(state.active_hosts) == 0 + + def test_missing_image_and_host(self): + with self.assertRaises(InventoryError): + make_inventory(hosts=("@podmanssh",)) + + def test_missing_image(self): + with self.assertRaises(InventoryError): + make_inventory(hosts=("@podmanssh/host",)) + + with self.assertRaises(InventoryError): + make_inventory(hosts=("@podmanssh/host:",)) + + def test_connect_all(self): + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + state = State(inventory, Config()) + connect_all(state) + assert len(state.active_hosts) == 1 + + def test_user_provided_container_id(self): + inventory = make_inventory( + hosts=(("@podmanssh/somehost:not-an-image", {"docker_container_id": "abc"}),), + ) + State(inventory, Config()) + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.connect() + assert host.data.docker_container_id == "abc" + + def test_connect_all_error(self): + inventory = make_inventory(hosts=("@podmanssh/somehost:a-broken-image",)) + state = State(inventory, Config()) + + with self.assertRaises(PyinfraError): + connect_all(state) + + def test_connect_disconnect_host(self): + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + state = State(inventory, Config()) + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.connect(reason=True) + assert len(state.active_hosts) == 0 + host.disconnect() + + def test_run_shell_command(self): + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + State(inventory, Config()) + + command = "echo hi" + + fake_ssh_podman_shell.custom_command = [get_podman_command(command), True, []] + + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.connect() + out = host.run_shell_command( + command, + _stdin="hello", + _get_pty=True, + print_output=True, + ) + assert len(out) == 2 + assert out[0] is True + assert fake_ssh_podman_shell.ran_custom_command + + def test_run_shell_command_error(self): + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + state = State(inventory, Config()) + + command = "echo hi" + fake_ssh_podman_shell.custom_command = [get_podman_command(command), False, []] + + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.connect(state) + out = host.run_shell_command(command, _get_pty=True) + assert out[0] is False + assert fake_ssh_podman_shell.ran_custom_command + + @patch("pyinfra.connectors.dockerssh.mkstemp", lambda: (None, "local_tempfile")) + @patch("pyinfra.connectors.docker.os.close", lambda f: None) + @patch("pyinfra.connectors.ssh.SSHConnector.put_file") + def test_put_file(self, fake_put_file): + fake_ssh_podman_shell.custom_command = [ + "podman cp remote_tempfile containerid:not-another-file", + True, + [], + ] + + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + State(inventory, Config()) + + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.get_temp_filename = lambda _: "remote_tempfile" + host.connect() + + host.put_file("not-a-file", "not-another-file", print_output=True) + + # ensure copy from local to remote host + fake_put_file.assert_called_with("local_tempfile", "remote_tempfile") + + # ensure copy from remote host to remote podman container + assert fake_ssh_podman_shell.ran_custom_command + + @patch("pyinfra.connectors.ssh.SSHConnector.put_file") + def test_put_file_error(self, fake_put_file): + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + State(inventory, Config()) + + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.get_temp_filename = lambda _: "remote_tempfile" + host.connect() + + # SSH error + fake_put_file.return_value = False + with self.assertRaises(IOError): + host.put_file("not-a-file", "not-another-file", print_output=True) + + # podman copy error + fake_ssh_podman_shell.custom_command = [ + "podman cp remote_tempfile containerid:not-another-file", + False, + CommandOutput([OutputLine("stderr", "podman error")]), + ] + fake_put_file.return_value = True + + with self.assertRaises(IOError) as e: + host.put_file("not-a-file", "not-another-file", print_output=True) + assert str(e.exception) == "podman error" + + @patch("pyinfra.connectors.ssh.SSHConnector.get_file") + def test_get_file(self, fake_get_file): + fake_ssh_podman_shell.custom_command = [ + "podman cp containerid:not-a-file remote_tempfile", + True, + [], + ] + + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + State(inventory, Config()) + + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.get_temp_filename = lambda _: "remote_tempfile" + host.connect() + + host.get_file("not-a-file", "not-another-file", print_output=True) + + # ensure copy from local to remote host + fake_get_file.assert_called_with("remote_tempfile", "not-another-file") + + # ensure copy from remote host to remote podman container + assert fake_ssh_podman_shell.ran_custom_command + + @patch("pyinfra.connectors.ssh.SSHConnector.get_file") + def test_get_file_error(self, fake_get_file): + fake_ssh_podman_shell.custom_command = [ + "podman cp containerid:not-a-file remote_tempfile", + False, + CommandOutput([OutputLine("stderr", "podman error")]), + ] + + inventory = make_inventory(hosts=("@podmanssh/somehost:not-an-image",)) + State(inventory, Config()) + + host = inventory.get_host("@podmanssh/somehost:not-an-image") + host.get_temp_filename = lambda _: "remote_tempfile" + host.connect() + + fake_get_file.return_value = True + with self.assertRaises(IOError) as ex: + host.get_file("not-a-file", "not-another-file", print_output=True) + + assert str(ex.exception) == "podman error" + + # SSH error + fake_ssh_podman_shell.custom_command = [ + "podman cp containerid:not-a-file remote_tempfile", + True, + [], + ] + fake_get_file.return_value = False + with self.assertRaises(IOError) as ex: + host.get_file("not-a-file", "not-another-file", print_output=True) + + assert str(ex.exception) == "failed to copy file over ssh"