Merge pull request #9322 from ThomasWaldmann/linux-container-testing-master

linux-run: run commands (e.g. tox) in a podman linux container

Author: ThomasWaldmann

docs/development.rst

@@ -405,6 +405,50 @@ Usage::
    # To copy files from the VM (in this case, the generated binary):
    vagrant scp OS:/vagrant/borg/borg.exe .
 
+Using Podman
+------------
+
+macOS-based developers (and others who prefer containers) can run the Linux test suite locally using Podman.
+
+Prerequisites:
+
+-   Install Podman (e.g., ``brew install podman``).
+-   Initialize the Podman machine, only once: ``podman machine init``.
+-   Start the Podman machine, before using it: ``podman machine start``.
+
+Usage::
+
+    # Open an interactive shell in the container (default if no command given):
+    ./scripts/linux-run
+
+    # Run the default tox environment:
+    ./scripts/linux-run tox
+
+    # Run a specific tox environment:
+    ./scripts/linux-run tox -e py311-pyfuse3
+
+    # Pass arguments to pytest (e.g., run specific tests):
+    ./scripts/linux-run tox -e py313-pyfuse3 -- -k mount
+
+    # Switch base image (temporarily):
+    ./scripts/linux-run --image python:3.11-bookworm tox
+
+Resource Usage
+~~~~~~~~~~~~~~
+
+The default Podman VM uses 2GB RAM and half your CPUs.
+For heavy tests (parallel execution), this might be tight.
+
+-   **Check usage:** Run ``podman stats`` in another terminal while tests are running.
+-   **Increase resources:**
+
+    ::
+
+        podman machine stop
+        podman machine set --cpus 6 --memory 4096
+        podman machine start
+
+
 Creating standalone binaries
 ----------------------------
 

scripts/Dockerfile.linux-tox

@@ -0,0 +1,30 @@
+ARG BASE_IMAGE=python:3.13
+FROM ${BASE_IMAGE}
+
+# Install system dependencies
+# These match the dependencies installed in CI for Linux (Debian/Ubuntu)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    pkg-config \
+    libssl-dev \
+    libacl1-dev \
+    libxxhash-dev \
+    liblz4-dev \
+    libzstd-dev \
+    libfuse3-dev \
+    fuse3 \
+    python3-dev \
+    git \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install tox
+RUN pip install tox
+
+# Set working directory
+WORKDIR /app
+
+# Check that we have the expected python version
+RUN python3 --version
+
+# Default command (can be overridden)
+CMD ["tox"]

scripts/linux-run

@@ -0,0 +1,110 @@
+#!/bin/bash
+# run commands in a linux container, e.g. for testing - for more info, see docs/development.rst.
+
+set -euo pipefail
+# set -x # Uncomment for debugging
+
+# Default configuration
+BASE_IMAGE="python:3.13"
+IMAGE_NAME="borg-test-env"
+CONTAINER_NAME="borg-test-runner"
+
+usage() {
+    echo "Usage: $0 [options] [command [args...]]"
+    echo ""
+    echo "Options:"
+    echo "  --image IMAGE    Base Docker image to use (default: $BASE_IMAGE)"
+    echo "  --rebuild        Force rebuild of the container image"
+    echo "  --help           Show this help message"
+    echo ""
+    echo "Examples:"
+    echo "  $0                                       # Open interactive shell (default)"
+    echo "  $0 tox -e py313-pyfuse3                  # Run tox in the container"
+    echo "  $0 --image python:3.11 tox -e py311-none # Run with specific image"
+    echo "  $0 ls -la                                # Run arbitrary command"
+    exit 1
+}
+
+# Parse specific arguments
+REBUILD=false
+
+# We use an array to store the command and its arguments.
+COMMAND=()
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --image)
+            BASE_IMAGE="$2"
+            shift 2
+            ;;
+        --rebuild)
+            REBUILD=true
+            shift
+            ;;
+        --help)
+            usage
+            ;;
+        --)
+            # Stop option parsing and treat the rest as command
+            shift
+            COMMAND+=("$@")
+            break
+            ;;
+        *)
+            # Stop option parsing and treat this and the rest as command
+            COMMAND+=("$@")
+            break
+            ;;
+    esac
+done
+
+# Ensure Podman is available
+if ! command -v podman &> /dev/null; then
+    echo "Error: podman is not installed. Please run 'brew install podman' and initialize it."
+    exit 1
+fi
+
+# Build the image if needed or requested.
+# We tag the image with the base image name to allow multiple versions.
+TAG_SUFFIX=$(echo "$BASE_IMAGE" | tr ':' '-')
+FULL_IMAGE_NAME="${IMAGE_NAME}:${TAG_SUFFIX}"
+
+if [[ "$REBUILD" == "true" ]] || ! podman image exists "$FULL_IMAGE_NAME"; then
+    echo "Building test image based on $BASE_IMAGE..."
+    podman build \
+        --build-arg BASE_IMAGE="$BASE_IMAGE" \
+        -t "$FULL_IMAGE_NAME" \
+        -f scripts/Dockerfile.linux-tox .
+fi
+
+echo "Running in container..."
+echo "Base Image: $BASE_IMAGE"
+if [[ ${#COMMAND[@]} -gt 0 ]]; then
+    echo "Command: ${COMMAND[*]}"
+else
+    echo "Command: /bin/bash (default)"
+fi
+
+# Run the container
+# --userns=keep-id: Maps the container user to the host user (Crucial for macOS volume mounts)
+# --security-opt label=disable: Disables SELinux separation if active (harmless on macOS)
+# --device /dev/fuse: Required for FUSE filesystem support (borg mount tests)
+# --volume $(pwd):/app: Mounts current directory to /app
+# --mount type=volume,dst=/tmp: Using a volume for /tmp to prevent issues with tmp files from different users
+
+CMD=(podman run --rm -it \
+    --name "$CONTAINER_NAME" \
+    --userns=keep-id \
+    --cap-add SYS_ADMIN \
+    --security-opt label=disable \
+    --device /dev/fuse \
+    --volume "$(pwd):/app" \
+    --mount type=volume,dst=/tmp \
+    --workdir /app \
+    "$FULL_IMAGE_NAME")
+
+if [[ ${#COMMAND[@]} -gt 0 ]]; then
+    "${CMD[@]}" "${COMMAND[@]}"
+else
+    "${CMD[@]}" /bin/bash
+fi