diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efd612ab5b..94c4c8b863 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,3 +254,182 @@ jobs: name: ${{ matrix.binary }} path: artifacts/* if-no-files-found: error + + vm_tests: + + # Cross-OS tests running inside VMs, aligned with master branch structure. + # Uses cross-platform-actions/action to run on FreeBSD, NetBSD, OpenBSD, Haiku. + permissions: + contents: read + id-token: write + attestations: write + runs-on: ubuntu-24.04 + timeout-minutes: 90 + needs: [lint] + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - os: freebsd + version: '14.3' + display_name: FreeBSD + # Controls binary build and provenance attestation on tags + do_binaries: true + artifact_prefix: borg-freebsd-14-x86_64-gh + - os: netbsd + version: '10.1' + display_name: NetBSD + do_binaries: false + - os: openbsd + version: '7.7' + display_name: OpenBSD + do_binaries: false + - os: haiku + version: 'r1beta5' + display_name: Haiku + do_binaries: false + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Test on ${{ matrix.display_name }} + id: cross_os + uses: cross-platform-actions/action@v0.29.0 + env: + DO_BINARIES: ${{ matrix.do_binaries }} + with: + operating_system: ${{ matrix.os }} + version: ${{ matrix.version }} + shell: bash + run: | + set -euxo pipefail + case "${{ matrix.os }}" in + freebsd) + export IGNORE_OSVERSION=yes + sudo -E pkg update -f + sudo -E pkg install -y xxhash liblz4 zstd pkgconf + # Install one of the FUSE libraries; fail if neither is available + sudo -E pkg install -y fusefs-libs || sudo -E pkg install -y fusefs-libs3 + sudo -E pkg install -y rust + sudo -E pkg install -y git + sudo -E pkg install -y python310 py310-sqlite3 + sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv + sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3 + sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python + sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3 + sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip + python -m venv .venv + . .venv/bin/activate + python -V + pip -V + python -m pip install --upgrade pip wheel + pip install -r requirements.d/development.txt + pip install -e . + tox -e py311-none + if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then + python -m pip install 'pyinstaller==6.14.2' + mkdir -p dist/binary + pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec + pushd dist/binary + echo "single-file binary" + chmod +x borg.exe + ./borg.exe -V + echo "single-directory binary" + chmod +x borg-dir/borg.exe + ./borg-dir/borg.exe -V + tar czf borg.tgz borg-dir + popd + mkdir -p artifacts + if [ -f dist/binary/borg.exe ]; then + cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }} + fi + if [ -f dist/binary/borg.tgz ]; then + cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz + fi + fi + ;; + netbsd) + arch="$(uname -m)" + sudo -E mkdir -p /usr/pkg/etc/pkgin + echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null + sudo -E pkgin update + sudo -E pkgin -y upgrade + sudo -E pkgin -y install zstd lz4 xxhash git + sudo -E pkgin -y install rust + sudo -E pkgin -y install pkg-config + sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox + sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 + sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 + sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 + sudo -E ln -sf /usr/pkg/bin/tox-3.11 /usr/pkg/bin/tox3 + # Ensure base system admin tools are on PATH for the non-root shell + export PATH="/sbin:/usr/sbin:$PATH" + echo "--- Preparing an extattr-enabled filesystem ---" + # On many NetBSD setups /tmp is tmpfs without extended attributes. + # Create a FFS image with extended attributes enabled and use it for TMPDIR. + VNDDEV="vnd0" + IMGFILE="/tmp/fs.img" + sudo -E dd if=/dev/zero of=${IMGFILE} bs=1m count=1024 + sudo -E vndconfig -c "${VNDDEV}" "${IMGFILE}" + sudo -E newfs -O 2ea /dev/r${VNDDEV}a + MNT="/mnt/eafs" + sudo -E mkdir -p ${MNT} + sudo -E mount -t ffs -o extattr /dev/${VNDDEV}a $MNT + export TMPDIR="${MNT}/tmp" + sudo -E mkdir -p ${TMPDIR} + sudo -E chmod 1777 ${TMPDIR} + touch ${TMPDIR}/testfile + lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***" + tox3 -e py311-none + ;; + openbsd) + sudo -E pkg_add xxhash lz4 zstd git + sudo -E pkg_add rust + sudo -E pkg_add openssl%3.4 + sudo -E pkg_add py3-pip py3-virtualenv py3-tox + export BORG_OPENSSL_NAME=eopenssl34 + tox -e py312-none + ;; + haiku) + pkgman refresh + pkgman install -y git pkgconfig zstd lz4 xxhash + pkgman install -y openssl3 + pkgman install -y rust_bin + pkgman install -y python3.10 + pkgman install -y cffi + pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel + # there is no pkgman package for tox, so we install it into a venv + python3 -m ensurepip --upgrade + python3 -m pip install --upgrade pip wheel + python3 -m venv .venv + . .venv/bin/activate + export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + export BORG_LIBLZ4_PREFIX=/system/develop + export BORG_LIBZSTD_PREFIX=/system/develop + export BORG_LIBXXHASH_PREFIX=/system/develop + export BORG_OPENSSL_PREFIX=/system/develop + pip install -r requirements.d/development.txt + pip install -e . + # troubles with either tox or pytest xdist, so we run pytest manually: + pytest -v -rs --benchmark-skip -k "not remote and not socket" + ;; + esac + + - name: Upload artifacts + if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.version }}-dist + path: artifacts/* + if-no-files-found: ignore + + - name: Attest provenance + if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries + uses: actions/attest-build-provenance@v3 + with: + subject-path: 'artifacts/*' diff --git a/src/borg/platformflags.py b/src/borg/platformflags.py index 2c1567f268..41ec94681e 100644 --- a/src/borg/platformflags.py +++ b/src/borg/platformflags.py @@ -9,4 +9,6 @@ is_win32 = sys.platform.startswith('win32') is_linux = sys.platform.startswith('linux') is_freebsd = sys.platform.startswith('freebsd') +is_netbsd = sys.platform.startswith('netbsd') +is_openbsd = sys.platform.startswith('openbsd') is_darwin = sys.platform.startswith('darwin') diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index 87967b552a..72a003d7fd 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -30,6 +30,8 @@ # Does this version of llfuse support ns precision? have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') if llfuse else False +has_mknod = hasattr(os, 'mknod') + try: from pytest import raises except: # noqa diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index cf4bba3d16..09d1c1dac8 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -56,10 +56,10 @@ from ..logger import setup_logging from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository -from . import has_lchflags, llfuse +from . import has_lchflags, has_mknod, llfuse from . import BaseTestCase, changedir, environment_variable, no_selinux, same_ts_ns, granularity_sleep from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported -from .platform import fakeroot_detected, is_darwin, is_freebsd, is_win32 +from .platform import fakeroot_detected, is_darwin, is_freebsd, is_netbsd, is_win32 from .upgrader import make_attic_repo from . import key @@ -367,10 +367,11 @@ def create_test_files(self, create_hardlinks=True): if has_lchflags: platform.set_flags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP) try: - # Block device - os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20)) - # Char device - os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40)) + if has_mknod: + # Block device + os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20)) + # Char device + os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40)) # File mode os.chmod('input/dir2', 0o555) # if we take away write perms, we need root to remove contents # File owner @@ -426,8 +427,8 @@ def test_basic_functionality(self): expected.append('input/link1') if are_hardlinks_supported(): expected.append('input/hardlink') - if not have_root: - # we could not create these device files without (fake)root + if not have_root or not has_mknod: + # we could not create these device files without (fake)root or without os.mknod expected.remove('input/bdev') expected.remove('input/cdev') if has_lchflags: @@ -4879,7 +4880,10 @@ def get_changes(filename, data): unexpected = {'type': 'modified', 'added': 0, 'removed': 0} assert unexpected not in get_changes('input/file_touched', joutput) if not content_only: - assert {"ctime", "mtime"}.issubset({c["type"] for c in get_changes('input/file_touched', joutput)}) + # On Windows, ctime is the creation time and does not change on touch. + # NetBSD also only reports mtime here, see #8703 (backport of #9161 intent). + expected = {"mtime"} if (is_win32 or is_netbsd) else {"ctime", "mtime"} + assert expected.issubset({c["type"] for c in get_changes('input/file_touched', joutput)}) else: # And if we're doing content-only, don't show the file at all. assert not any(get_changes('input/file_touched', joutput)) @@ -5074,6 +5078,10 @@ def test_time_diffs(self): @requires_hardlinks + @pytest.mark.skipif( + (not are_hardlinks_supported()) or is_freebsd or is_netbsd, + reason='Skip when hardlinks unsupported or on FreeBSD/NetBSD due to differing ctime/link handling; see #9147, #9153.', + ) def test_multiple_link_exclusion(self): path_a = os.path.join(self.input_path, 'a') path_b = os.path.join(self.input_path, 'b') diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 54f56e66e5..627969891c 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -6,7 +6,7 @@ import tempfile import unittest -from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin +from ..platformflags import is_win32, is_linux, is_freebsd, is_netbsd, is_darwin from ..platform import acl_get, acl_set, swidth from ..platform import get_process_id, process_alive from . import BaseTestCase, unopened_tempfile