From 8e072953fe09aff205e8663eff4bbab03114271f Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Mon, 15 Sep 2025 22:08:49 +1000 Subject: [PATCH 01/13] remove unused logs --- src/restic_compose_backup/log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/restic_compose_backup/log.py b/src/restic_compose_backup/log.py index 078fe86..ceaff0c 100644 --- a/src/restic_compose_backup/log.py +++ b/src/restic_compose_backup/log.py @@ -1,9 +1,7 @@ import logging -import os import sys logger = logging.getLogger("restic_compose_backup") -HOSTNAME = os.environ["HOSTNAME"] DEFAULT_LOG_LEVEL = logging.INFO LOG_LEVELS = { @@ -22,7 +20,5 @@ def setup(level: str = "warning"): ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(level) - # ch.setFormatter(logging.Formatter('%(asctime)s - {HOSTNAME} - %(name)s - %(levelname)s - %(message)s')) - # ch.setFormatter(logging.Formatter('%(asctime)s - {HOSTNAME} - %(levelname)s - %(message)s')) ch.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")) logger.addHandler(ch) From 508639511310a8f8e386e6c1bb83b565e0f43bc4 Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Mon, 15 Sep 2025 22:38:16 +1000 Subject: [PATCH 02/13] gracefully handle signals --- src/entrypoint.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 3098b07..2e262cc 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -6,6 +6,13 @@ rcb dump-env > /.env # Write crontab rcb crontab > crontab -# start cron in the foreground +# Start cron in the background and capture its PID crontab crontab -crond -f +crond -f & +CRON_PID=$! + +# Trap termination signals and kill the cron process +trap 'kill $CRON_PID; exit 0' TERM INT + +# Wait for cron and handle signals +wait $CRON_PID From f29307a263db6f7f995121badfbf37ec85dee8af Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Tue, 16 Sep 2025 08:17:39 +1000 Subject: [PATCH 03/13] created backup container just uses own stack --- src/restic_compose_backup/backup_runner.py | 3 --- src/restic_compose_backup/cli.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/restic_compose_backup/backup_runner.py b/src/restic_compose_backup/backup_runner.py index 7f94e0b..5760081 100644 --- a/src/restic_compose_backup/backup_runner.py +++ b/src/restic_compose_backup/backup_runner.py @@ -12,7 +12,6 @@ def run( volumes: dict = None, environment: dict = None, labels: dict = None, - source_container_id: str = None, ): logger.info("Starting backup container") client = utils.docker_client() @@ -21,11 +20,9 @@ def run( image, command, labels=labels, - # auto_remove=True, # We remove the container further down detach=True, environment=environment + ["BACKUP_PROCESS_CONTAINER=true"], volumes=volumes, - network_mode=f"container:{source_container_id}", # Reuse original container's network stack. working_dir=os.getcwd(), tty=True, ) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index 5961c8d..1269a07 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -169,10 +169,8 @@ def backup(config, containers): command="rcb start-backup-process", volumes=volumes, environment=containers.this_container.environment, - source_container_id=containers.this_container.id, labels={ containers.backup_process_label: "True", - "com.docker.compose.project": containers.project_name, }, ) except Exception as ex: From c95caf60734d3b2061d5013fd948f15cf64075af Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Tue, 16 Sep 2025 08:34:31 +1000 Subject: [PATCH 04/13] switch to use socket lib to get hostname --- src/restic_compose_backup/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index eaea70a..5706798 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -1,6 +1,6 @@ -import os import logging from pathlib import Path +import socket from typing import List from restic_compose_backup import enums, utils @@ -407,7 +407,7 @@ def __init__(self): # Find the container we are running in. # If we don't have this information we cannot continue for container_data in all_containers: - if container_data.get("Id").startswith(os.environ["HOSTNAME"]): + if container_data.get("Id").startswith(socket.gethostname()): self.this_container = Container(container_data) if not self.this_container: From d6f00d3b99ecdc8e80f2e0499c52c59dc65de9bd Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Tue, 16 Sep 2025 08:51:43 +1000 Subject: [PATCH 05/13] Add back ability to run over all projects --- src/restic_compose_backup/config.py | 1 + src/restic_compose_backup/containers.py | 17 ++++++----------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/restic_compose_backup/config.py b/src/restic_compose_backup/config.py index 31411bb..75d51d3 100644 --- a/src/restic_compose_backup/config.py +++ b/src/restic_compose_backup/config.py @@ -29,6 +29,7 @@ def __init__(self, check=True): self.swarm_mode = os.environ.get("SWARM_MODE") or False self.include_project_name = os.environ.get("INCLUDE_PROJECT_NAME") or False self.exclude_bind_mounts = os.environ.get("EXCLUDE_BIND_MOUNTS") or False + self.include_all_compose_projects = os.environ.get("INCLUDE_ALL_COMPOSE_PROJECTS") or False self.include_all_volumes = os.environ.get("INCLUDE_ALL_VOLUMES") or False if self.include_all_volumes: logger.warning( diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index 5706798..bfa7b20 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -413,7 +413,7 @@ def __init__(self): if not self.this_container: raise ValueError("Cannot find metadata for backup container") - # Gather all running containers in the current compose setup + # Gather relevant containers for container_data in all_containers: container = Container(container_data) @@ -429,24 +429,19 @@ def __init__(self): if not container.is_running: continue + # If not swarm mode we need to filter in compose project + if not config.swarm_mode and not config.include_all_compose_projects and container.project_name != self.this_container.project_name: + continue + # Gather stop during backup containers if container.stop_during_backup: - if config.swarm_mode: - self.stop_during_backup_containers.append(container) - else: - if container.project_name == self.this_container.project_name: - self.stop_during_backup_containers.append(container) + self.stop_during_backup_containers.append(container) # Detect running backup process container if container.is_backup_process_container: self.backup_process_container = container - # --- Determine what containers should be evaluated - # If not swarm mode we need to filter in compose project - if not config.swarm_mode: - if container.project_name != self.this_container.project_name: - continue # Containers started manually are not included if container.is_oneoff: From 6ce96afbfe1543a1bbd8a43636341263c32f82d9 Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Tue, 16 Sep 2025 17:55:02 +1000 Subject: [PATCH 06/13] add back label --- src/restic_compose_backup/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index 1269a07..758c957 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -171,6 +171,7 @@ def backup(config, containers): environment=containers.this_container.environment, labels={ containers.backup_process_label: "True", + "com.docker.compose.project": containers.project_name, }, ) except Exception as ex: From 5823127ba497bc5eed47afc614f6f43606976bcc Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 06:56:22 +1000 Subject: [PATCH 07/13] update comment to match command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a004ec..a80f142 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Contributions are welcome regardless of experience level. ## Python environment -Use [`uv`](https://docs.astral.sh/uv/) within the `src/` directory to manage your development environment. +Use [`uv`](https://docs.astral.sh/uv/) within the the repo root directory to manage your development environment. ```bash git clone https://github.com/lawndoc/stack-back.git From f59f7b5fbe95726a5eb6927212576722a12e2b16 Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 07:06:58 +1000 Subject: [PATCH 08/13] remove unused import --- src/tests/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/tests.py b/src/tests/tests.py index 1d0b452..cfe0564 100644 --- a/src/tests/tests.py +++ b/src/tests/tests.py @@ -1,4 +1,3 @@ -import json import os import unittest from unittest import mock From f53acc6279b77e8e7be2f44b0afd8f3fbcb22279 Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 07:08:09 +1000 Subject: [PATCH 09/13] ruff --- src/restic_compose_backup/config.py | 4 +++- src/restic_compose_backup/containers.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/restic_compose_backup/config.py b/src/restic_compose_backup/config.py index 75d51d3..27deede 100644 --- a/src/restic_compose_backup/config.py +++ b/src/restic_compose_backup/config.py @@ -29,7 +29,9 @@ def __init__(self, check=True): self.swarm_mode = os.environ.get("SWARM_MODE") or False self.include_project_name = os.environ.get("INCLUDE_PROJECT_NAME") or False self.exclude_bind_mounts = os.environ.get("EXCLUDE_BIND_MOUNTS") or False - self.include_all_compose_projects = os.environ.get("INCLUDE_ALL_COMPOSE_PROJECTS") or False + self.include_all_compose_projects = ( + os.environ.get("INCLUDE_ALL_COMPOSE_PROJECTS") or False + ) self.include_all_volumes = os.environ.get("INCLUDE_ALL_VOLUMES") or False if self.include_all_volumes: logger.warning( diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index bfa7b20..f8f7b07 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -430,7 +430,11 @@ def __init__(self): continue # If not swarm mode we need to filter in compose project - if not config.swarm_mode and not config.include_all_compose_projects and container.project_name != self.this_container.project_name: + if ( + not config.swarm_mode + and not config.include_all_compose_projects + and container.project_name != self.this_container.project_name + ): continue # Gather stop during backup containers @@ -441,8 +445,6 @@ def __init__(self): if container.is_backup_process_container: self.backup_process_container = container - - # Containers started manually are not included if container.is_oneoff: continue From 2da8dca7beb0124098f46aca766f8a4b53ff5009 Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 09:53:28 +1000 Subject: [PATCH 10/13] add hostname mocking --- src/tests/tests.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/tests/tests.py b/src/tests/tests.py index cfe0564..75cf397 100644 --- a/src/tests/tests.py +++ b/src/tests/tests.py @@ -15,16 +15,20 @@ class BaseTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - """Set up basic environment variables""" - # os.environ['RESTIC_REPOSITORY'] = "test" - # os.environ['RESTIC_PASSWORD'] = "password" + cls.backup_hash = fixtures.generate_sha256() + cls.hostname = cls.backup_hash[:8] + + cls.hostname_patcher = mock.patch("socket.gethostname", return_value=cls.hostname) + cls.hostname_patcher.start() + + @classmethod + def tearDownClass(cls): + cls.hostname_patcher.stop() def createContainers(self): - backup_hash = fixtures.generate_sha256() - os.environ["HOSTNAME"] = backup_hash[:8] return [ { - "id": backup_hash, + "id": self.backup_hash, "service": "backup", } ] @@ -376,10 +380,12 @@ def test_stop_container_during_backup_database(self): class IncludeAllVolumesTests(BaseTestCase): @classmethod def setUpClass(cls): + super().setUpClass() config.config.auto_backup_all = "true" @classmethod def tearDownClass(cls): + super().tearDownClass() config.config = config.Config() def test_all_volumes(self): From c504b3ec6399ce7ddbfb4eaada22035684b65e40 Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 09:55:47 +1000 Subject: [PATCH 11/13] ruff --- src/tests/tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/tests.py b/src/tests/tests.py index 75cf397..5775adf 100644 --- a/src/tests/tests.py +++ b/src/tests/tests.py @@ -16,9 +16,10 @@ class BaseTestCase(unittest.TestCase): @classmethod def setUpClass(cls): cls.backup_hash = fixtures.generate_sha256() - cls.hostname = cls.backup_hash[:8] - cls.hostname_patcher = mock.patch("socket.gethostname", return_value=cls.hostname) + cls.hostname_patcher = mock.patch( + "socket.gethostname", return_value=cls.backup_hash[:8] + ) cls.hostname_patcher.start() @classmethod From 164fda02b799a8d20a4590967cde3c123875ce7a Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 10:41:20 +1000 Subject: [PATCH 12/13] switch to use same network not same interface --- src/restic_compose_backup/backup_runner.py | 2 ++ src/restic_compose_backup/cli.py | 3 ++- src/restic_compose_backup/containers.py | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/restic_compose_backup/backup_runner.py b/src/restic_compose_backup/backup_runner.py index 5760081..6f7b1d2 100644 --- a/src/restic_compose_backup/backup_runner.py +++ b/src/restic_compose_backup/backup_runner.py @@ -12,6 +12,7 @@ def run( volumes: dict = None, environment: dict = None, labels: dict = None, + source_container_network: str = None, ): logger.info("Starting backup container") client = utils.docker_client() @@ -23,6 +24,7 @@ def run( detach=True, environment=environment + ["BACKUP_PROCESS_CONTAINER=true"], volumes=volumes, + network=source_container_network, working_dir=os.getcwd(), tty=True, ) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index 758c957..9b45ebb 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -138,7 +138,7 @@ def status(config, containers): logger.info("-" * 67) -def backup(config, containers): +def backup(config, containers: RunningContainers): """Request a backup to start""" # Make sure we don't spawn multiple backup processes if containers.backup_process_running: @@ -169,6 +169,7 @@ def backup(config, containers): command="rcb start-backup-process", volumes=volumes, environment=containers.this_container.environment, + source_container_network=containers.this_container.network, labels={ containers.backup_process_label: "True", "com.docker.compose.project": containers.project_name, diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index f8f7b07..b203910 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -56,6 +56,14 @@ def id(self) -> str: """str: The id of the container""" return self._data.get("Id") + @property + def network(self) -> str: + """str: The name of the network the container is connected to""" + network_settings = self._data.get("NetworkSettings", {}) + first_network = list(network_settings.get("Networks", {}).values())[0] + network_name = first_network.get("NetworkID", "") + return network_name + @property def hostname(self) -> str: """Hostname of the container""" From 36ec179e672a30aeff2d4cb677e1ecd92b0606ad Mon Sep 17 00:00:00 2001 From: Alex McDermott Date: Sat, 27 Sep 2025 19:57:36 +1000 Subject: [PATCH 13/13] switch to use ip for db connection --- src/restic_compose_backup/cli.py | 2 +- src/restic_compose_backup/containers.py | 21 +++++++++++++-------- src/restic_compose_backup/containers_db.py | 6 +++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index 9b45ebb..05a3a11 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -169,7 +169,7 @@ def backup(config, containers: RunningContainers): command="rcb start-backup-process", volumes=volumes, environment=containers.this_container.environment, - source_container_network=containers.this_container.network, + source_container_network=containers.this_container.network_name, labels={ containers.backup_process_label: "True", "com.docker.compose.project": containers.project_name, diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index b203910..47e4a1a 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -57,17 +57,22 @@ def id(self) -> str: return self._data.get("Id") @property - def network(self) -> str: + def network_details(self) -> dict: + """dict: The network details of the container""" + network_settings: dict = self._data.get("NetworkSettings", {}) + networks: dict = network_settings.get("Networks", {}) + first_network = list(networks.values())[0] + return first_network + + @property + def network_name(self) -> str: """str: The name of the network the container is connected to""" - network_settings = self._data.get("NetworkSettings", {}) - first_network = list(network_settings.get("Networks", {}).values())[0] - network_name = first_network.get("NetworkID", "") - return network_name + return self.network_details.get("NetworkID", "") @property - def hostname(self) -> str: - """Hostname of the container""" - return self.get_config("Hostname", default=self.id[0:12]) + def ip_address(self) -> str: + """str: IP address of the container""" + return self.network_details.get("IPAddress", "") @property def image(self) -> str: diff --git a/src/restic_compose_backup/containers_db.py b/src/restic_compose_backup/containers_db.py index 7f43532..d477bf1 100644 --- a/src/restic_compose_backup/containers_db.py +++ b/src/restic_compose_backup/containers_db.py @@ -21,7 +21,7 @@ def get_credentials(self) -> dict: username = self.get_config_env("MARIADB_USER") password = self.get_config_env("MARIADB_PASSWORD") return { - "host": self.hostname, + "host": self.ip_address, "username": username, "password": password, "port": "3306", @@ -91,7 +91,7 @@ def get_credentials(self) -> dict: username = self.get_config_env("MYSQL_USER") password = self.get_config_env("MYSQL_PASSWORD") return { - "host": self.hostname, + "host": self.ip_address, "username": username, "password": password, "port": "3306", @@ -155,7 +155,7 @@ class PostgresContainer(Container): def get_credentials(self) -> dict: """dict: get credentials for the service""" return { - "host": self.hostname, + "host": self.ip_address, "username": self.get_config_env("POSTGRES_USER"), "password": self.get_config_env("POSTGRES_PASSWORD"), "port": "5432",