diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe1cd92f33..21e8e6c3aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 25 - needs: [security] + needs: [lint, security] steps: - uses: actions/checkout@v4 @@ -120,7 +120,7 @@ jobs: echo "Using LD_PRELOAD=$LD_PRELOAD" pytest -v --benchmark-skip -k "not remote" - posix_tests: + native_tests: needs: [lint, security] permissions: @@ -328,12 +328,197 @@ jobs: path: artifacts/* if-no-files-found: error + vm_tests: + permissions: + contents: read + id-token: write + attestations: write + runs-on: ubuntu-24.04 + timeout-minutes: 90 + needs: [lint, security] + continue-on-error: true + + strategy: + fail-fast: false + matrix: + include: + - os: freebsd + version: '14.3' + display_name: FreeBSD + # Controls binary build and provenance attestation on tags + do_binaries: true + artifact_prefix: borg-freebsd-14-x86_64-gh + + - os: netbsd + version: '10.1' + display_name: NetBSD + do_binaries: false + + - os: openbsd + version: '7.7' + display_name: OpenBSD + do_binaries: false + + - os: haiku + version: 'r1beta5' + display_name: Haiku + do_binaries: false + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Test on ${{ matrix.display_name }} + id: cross_os + uses: cross-platform-actions/action@v0.29.0 + env: + DO_BINARIES: ${{ matrix.do_binaries }} + with: + operating_system: ${{ matrix.os }} + version: ${{ matrix.version }} + shell: bash + run: | + set -euxo pipefail + + case "${{ matrix.os }}" in + freebsd) + export IGNORE_OSVERSION=yes + sudo -E pkg update -f + sudo -E pkg install -y xxhash liblz4 zstd pkgconf + # Install one of the FUSE libraries; fail if neither is available + sudo -E pkg install -y fusefs-libs || sudo -E pkg install -y fusefs-libs3 + sudo -E pkg install -y rust + sudo -E pkg install -y git + sudo -E pkg install -y python310 py310-sqlite3 + sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv + sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3 + sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python + sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3 + sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip + + python -m venv .venv + . .venv/bin/activate + python -V + pip -V + python -m pip install --upgrade pip wheel + pip install -r requirements.d/development.txt + pip install -e . + pytest -v -n auto --benchmark-skip + + if [[ "${DO_BINARIES}" == "true" && "${GITHUB_REF:-}" == refs/tags/* ]]; then + python -m pip install 'pyinstaller==6.14.2' + mkdir -p dist/binary + pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec + pushd dist/binary + echo "single-file binary" + chmod +x borg.exe + ./borg.exe -V + echo "single-directory binary" + chmod +x borg-dir/borg.exe + ./borg-dir/borg.exe -V + tar czf borg.tgz borg-dir + popd + mkdir -p artifacts + if [ -f dist/binary/borg.exe ]; then + cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }} + fi + if [ -f dist/binary/borg.tgz ]; then + cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz + fi + fi + ;; + + netbsd) + arch="$(uname -m)" + sudo -E mkdir -p /usr/pkg/etc/pkgin + echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null + sudo -E pkgin update || true + sudo -E pkgin -y upgrade || true + sudo -E pkgin -y install zstd lz4 xxhash git || true + sudo -E pkgin -y install rust || true + sudo -E pkgin -y install pkg-config || true + sudo -E pkgin -y install py311-pip py311-virtualenv || true + sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true + sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true + sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true + + python3 -m virtualenv .venv + . .venv/bin/activate + python -m pip install --upgrade pip + pip install -r requirements.d/development.txt + pip install -e . + pytest -v -n auto --benchmark-skip + ;; + + openbsd) + # echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null || true + sudo -E pkg_add xxhash || true + sudo -E pkg_add lz4 || true + sudo -E pkg_add zstd || true + sudo -E pkg_add git || true + sudo -E pkg_add rust || true + sudo -E pkg_add openssl%3.4 || true + sudo -E pkg_add py3-pip || true + sudo -E pkg_add py3-virtualenv || true + + export BORG_OPENSSL_NAME=eopenssl34 + python3 -m virtualenv .venv + . .venv/bin/activate + python -m pip install --upgrade pip + pip install -r requirements.d/development.txt + pip install -e . + pytest -v -n auto --benchmark-skip + ;; + + haiku) + pkgman refresh || true + pkgman install -y git pkgconfig zstd lz4 xxhash || true + pkgman install -y openssl3 || true + pkgman install -y rust_bin || true + pkgman install -y python3.10 || true + pkgman install -y cffi || true + python3 -V || true + pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true + export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}" + export BORG_LIBLZ4_PREFIX=/system/develop + export BORG_LIBZSTD_PREFIX=/system/develop + export BORG_LIBXXHASH_PREFIX=/system/develop + export BORG_OPENSSL_PREFIX=/system/develop + + python3 -m ensurepip --upgrade || true + python3 -m pip install --upgrade pip wheel + python3 -m venv .venv + . .venv/bin/activate + python -V + pip install -r requirements.d/development.txt + pip install -e . + pytest -v --benchmark-skip -k "not remote and not socket" + ;; + esac + + - name: Upload artifacts + if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.version }}-dist + path: artifacts/* + if-no-files-found: ignore + + - name: Attest provenance + if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries + uses: actions/attest-build-provenance@v3 + with: + subject-path: 'artifacts/*' + windows_tests: if: false # can be used to temporarily disable the build runs-on: windows-latest timeout-minutes: 120 - needs: posix_tests + needs: native_tests env: PY_COLORS: 1 diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml deleted file mode 100644 index cf443bc8d9..0000000000 --- a/.github/workflows/freebsd.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: CI FreeBSD - -on: - push: - branches: [ master ] - tags: - - '2.*' - pull_request: - branches: [ master ] - paths: - - '**.py' - - '**.pyx' - - '**.c' - - '**.h' - - '**.yml' - - '**.toml' - - '**.cfg' - - '**.ini' - - 'requirements.d/*' - - '!docs/**' - -jobs: - freebsd_tests: - name: FreeBSD tests and build - permissions: - contents: read - id-token: write - attestations: write - runs-on: ubuntu-24.04 - continue-on-error: true - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Run on FreeBSD - id: freebsd - uses: cross-platform-actions/action@v0.29.0 - with: - operating_system: freebsd - version: '14.3' - shell: bash - # Increase resources a bit for building Cython extensions - cpu_count: 3 - memory: 8G - run: | - set -euxo pipefail - # Update package catalog and install dependencies - export IGNORE_OSVERSION=yes - sudo -E pkg update -f - sudo -E pkg install -y xxhash liblz4 zstd pkgconf - sudo -E pkg install -y fusefs-libs || true - sudo -E pkg install -y fusefs-libs3 || true - sudo -E pkg install -y rust - sudo -E pkg install -y git # fakeroot causes lots of troubles on freebsd - sudo -E pkg install -y python310 py310-sqlite3 - sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv - - # Ensure python3 and pip point to the chosen version - sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3 || true - sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python || true - sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3 || true - sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip || true - - # Create and activate venv - python -m venv .venv - . .venv/bin/activate - - # Build prerequisites - python -V - pip -V - python -m pip install --upgrade pip wheel - # Install dev requirements and project in editable mode - pip install -r requirements.d/development.txt - pip install -e . - - # Run tests (skip benchmarks which are slow on CI) - pytest -v -n auto --benchmark-skip - - # Only build PyInstaller binaries for tags - if [[ "${GITHUB_REF:-}" == refs/tags/* ]]; then - python -m pip install 'pyinstaller==6.14.2' - mkdir -p dist/binary - pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec - # Smoke-test the built binaries and package dir-mode as tarball - pushd dist/binary - echo "single-file binary" - chmod +x borg.exe - ./borg.exe -V - echo "single-directory binary" - chmod +x borg-dir/borg.exe - ./borg-dir/borg.exe -V - tar czf borg.tgz borg-dir - popd - # Prepare artifacts - mkdir -p artifacts - if [ -f dist/binary/borg.exe ]; then - cp -v dist/binary/borg.exe artifacts/borg-freebsd-14-x86_64-gh - fi - if [ -f dist/binary/borg.tgz ]; then - cp -v dist/binary/borg.tgz artifacts/borg-freebsd-14-x86_64-gh.tgz - fi - fi - - - name: Upload FreeBSD artifacts - if: startsWith(github.ref, 'refs/tags/') - uses: actions/upload-artifact@v4 - with: - name: freebsd14-dist - path: artifacts/* - if-no-files-found: ignore - - - name: Attest provenance for FreeBSD artifacts - if: startsWith(github.ref, 'refs/tags/') - uses: actions/attest-build-provenance@v3 - with: - subject-path: 'artifacts/*' diff --git a/.github/workflows/haiku.yml b/.github/workflows/haiku.yml deleted file mode 100644 index d4a5e3953c..0000000000 --- a/.github/workflows/haiku.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: CI Haiku - -on: - push: - branches: [ master ] - tags: - - '2.*' - pull_request: - branches: [ master ] - paths: - - '**.py' - - '**.pyx' - - '**.c' - - '**.h' - - '**.yml' - - '**.toml' - - '**.cfg' - - '**.ini' - - 'requirements.d/*' - - '!docs/**' - -jobs: - haiku_tests: - name: Haiku tests - runs-on: ubuntu-24.04 - timeout-minutes: 90 - continue-on-error: true - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - # setuptools-scm needs full history for versioning - fetch-depth: 0 - fetch-tags: true - - - name: Run on Haiku - id: haiku - uses: cross-platform-actions/action@v0.29.0 - with: - operating_system: haiku - version: 'r1beta5' - shell: bash - cpu_count: 2 - memory: 6G - run: | - set -euxo pipefail - # Refresh repositories - pkgman refresh || true - # pkgman update -y || true - - # Base tools and libraries - pkgman install -y git pkgconfig zstd lz4 xxhash || true - pkgman install -y openssl3 || true - pkgman install -y rust_bin || true - pkgman install -y python3.10 || true - pkgman install -y cffi || true - python3 -V || true - # Development headers and pkg-config files needed for building C extensions - pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true - - # Ensure pkg-config can find .pc files - export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}" - - # Point Borg build to library/include prefixes (avoids pkg-config name mismatches) - export BORG_LIBLZ4_PREFIX=/system/develop - export BORG_LIBZSTD_PREFIX=/system/develop - export BORG_LIBXXHASH_PREFIX=/system/develop - export BORG_OPENSSL_PREFIX=/system/develop - - # Ensure pip is available - python3 -m ensurepip --upgrade || true - python3 -m pip install --upgrade pip wheel - - # Create and activate a virtualenv - python3 -m venv .venv - . .venv/bin/activate - - python -V - # Install development/test dependencies - pip install -r requirements.d/development.txt - # Build Borg (editable) - pip install -e . - # Run tests; skip benchmarks - # remote archiver tests are broken on Haiku - pytest -v --benchmark-skip -k "not remote and not socket" diff --git a/.github/workflows/netbsd.yml b/.github/workflows/netbsd.yml deleted file mode 100644 index 9dd38cbfd9..0000000000 --- a/.github/workflows/netbsd.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: CI NetBSD - -on: - push: - branches: [ master ] - tags: - - '2.*' - pull_request: - branches: [ master ] - paths: - - '**.py' - - '**.pyx' - - '**.c' - - '**.h' - - '**.yml' - - '**.toml' - - '**.cfg' - - '**.ini' - - 'requirements.d/*' - - '!docs/**' - -jobs: - netbsd_tests: - name: NetBSD tests - runs-on: ubuntu-24.04 - timeout-minutes: 90 - continue-on-error: true - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - # setuptools-scm needs full history for versioning - fetch-depth: 0 - fetch-tags: true - - - name: Run on NetBSD - id: netbsd - uses: cross-platform-actions/action@v0.29.0 - with: - operating_system: netbsd - version: '10.1' - shell: bash - cpu_count: 3 - memory: 6G - run: | - set -euxo pipefail - # Configure pkgin repository based on architecture. - arch="$(uname -m)" - sudo -E mkdir -p /usr/pkg/etc/pkgin - echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null - sudo -E pkgin update || true - sudo -E pkgin -y upgrade || true - - # Base tools and libraries - sudo -E pkgin -y install zstd lz4 xxhash git || true - sudo -E pkgin -y install rust || true - # Work around NetBSD 9.3 bug with .bash_profile in screen sessions - # echo "export PROMPT_COMMAND=" >> ~/.bash_profile || true - sudo -E pkgin -y install pkg-config || true - # Python 3.11 and tools - sudo -E pkgin -y install py311-pip py311-virtualenv || true - sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true - sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true - sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true - - # Create and activate a virtualenv - python3 -m virtualenv .venv - . .venv/bin/activate - python -m pip install --upgrade pip - # Install development/test dependencies - pip install -r requirements.d/development.txt - # Build Borg (editable) - pip install -e . - # Run tests; skip benchmarks - pytest -v -n auto --benchmark-skip diff --git a/.github/workflows/openbsd.yml b/.github/workflows/openbsd.yml deleted file mode 100644 index 534a454d16..0000000000 --- a/.github/workflows/openbsd.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: CI OpenBSD - -on: - push: - branches: [ master ] - tags: - - '2.*' - pull_request: - branches: [ master ] - paths: - - '**.py' - - '**.pyx' - - '**.c' - - '**.h' - - '**.yml' - - '**.toml' - - '**.cfg' - - '**.ini' - - 'requirements.d/*' - - '!docs/**' - -jobs: - openbsd_tests: - name: OpenBSD tests - runs-on: ubuntu-24.04 - timeout-minutes: 90 - continue-on-error: true - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - # setuptools-scm needs full history for versioning - fetch-depth: 0 - fetch-tags: true - - - name: Run on OpenBSD - id: openbsd - uses: cross-platform-actions/action@v0.29.0 - with: - operating_system: openbsd - version: '7.7' - shell: bash - cpu_count: 3 - memory: 6G - run: | - set -euxo pipefail - # On OpenBSD, use pkg_add and set installurl. Use sudo for privileged operations. - # echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null - # crypto / compression libs - sudo -E pkg_add xxhash || true - sudo -E pkg_add lz4 || true - sudo -E pkg_add zstd || true - # toolchain / VCS - sudo -E pkg_add git || true - sudo -E pkg_add rust || true - # OpenSSL 3.4 - sudo -E pkg_add openssl%3.4 || true - # Python tooling - sudo -E pkg_add py3-pip || true - sudo -E pkg_add py3-virtualenv || true - - export BORG_OPENSSL_NAME=eopenssl34 - # Create and activate a virtualenv - python3 -m virtualenv .venv - . .venv/bin/activate - python -m pip install --upgrade pip - # Install development/test dependencies - pip install -r requirements.d/development.txt - # Build Borg (editable) - pip install -e . - # Run tests; skip benchmarks - pytest -v -n auto --benchmark-skip