Merge branch 'master' into enhancements/issue-6656

Author: ThomasWaldmann

.github/workflows/ci.yml

@@ -65,16 +65,16 @@ jobs:
               python-version: '3.11'
               toxenv: py311-fuse3
             - os: macos-12
-              python-version: '3.9'
-              toxenv: py39-none  # note: no fuse testing, due to #6099, see also #6196.
+              python-version: '3.11'
+              toxenv: py311-none  # note: no fuse testing, due to #6099, see also #6196.
 
     env:
       # Configure pkg-config to use OpenSSL from Homebrew
       PKG_CONFIG_PATH: "/usr/local/opt/[email protected]/lib/pkgconfig:$PKG_CONFIG_PATH"
       TOXENV: ${{ matrix.toxenv }}
 
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 40
+    timeout-minutes: 60
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/codeql-analysis.yml

@@ -52,7 +52,7 @@ jobs:
         sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,4 +66,4 @@ jobs:
         pip3 install -r requirements.d/development.txt
         pip3 install -e .
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+repos:
+-   repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+    -   id: flake8
+        files: '(src|scripts|conftest.py)'

LICENSE

@@ -1,4 +1,4 @@
-Copyright (C) 2015-2022 The Borg Collective (see AUTHORS file)
+Copyright (C) 2015-2023 The Borg Collective (see AUTHORS file)
 Copyright (C) 2010-2014 Jonas Borgström <[email protected]>
 All rights reserved.
 

Vagrantfile

@@ -15,7 +15,8 @@ def packages_debianoid(user)
     apt-get -y -qq update
     apt-get -y -qq dist-upgrade
     # for building borgbackup and dependencies:
-    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev pkg-config
+    apt install -y pkg-config
+    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true
     apt install -y libfuse-dev fuse || true
     apt install -y libfuse3-dev fuse3 || true
     apt install -y locales || true
@@ -136,9 +137,95 @@ def packages_openindiana
   EOF
 end
 
+
+# Build and install borg dependencies from source
+def install_source_dependencies(user)
+  return <<-EOF
+    set -e -o pipefail
+
+    # Install in /usr/local
+    export PREFIX=/usr/local
+
+    # Make PKG_CONFIG_PATH explicit, even if /usr/local/lib/pkgconfig is enabled per default
+    export PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig
+    echo 'export PKG_CONFIG_PATH="'${PKG_CONFIG_PATH}'"' >> ~#{user}/.bash_profile
+
+    # All source packages integrate with pkg-config, remove any previous overrides
+    sed -i '/BORG_.*_PREFIX/d' ~#{user}/.bash_profile
+
+    # Setup pyenv to pick up the custom openssl version (python >= 3.9 requires openssl >= 1.1.1)
+    echo 'export PYTHON_CONFIGURE_OPTS="--with-openssl='"${PREFIX}"' --with-openssl-rpath=auto"' >> ~#{user}/.bash_profile
+    echo 'export LDFLAGS=-Wl,-rpath,'"${PREFIX}"'/lib' >> ~#{user}/.bash_profile
+
+    # Silence git warning about shallow clones
+    git config --global advice.detachedHead false
+
+    # libattr
+    VERSION_LIBATTR=2.5.1
+    curl -s -L https://download.savannah.nongnu.org/releases/attr/attr-${VERSION_LIBATTR}.tar.gz | tar xvz --strip-components=1 --one-top-level=attr -f - -C ${PREFIX}/src
+    cd ${PREFIX}/src/attr
+    ./configure --prefix=${PREFIX}
+    make -j$(nproc) install
+
+    # libacl
+    VERSION_LIBACL=2.3.1
+    curl -s -L https://download.savannah.nongnu.org/releases/acl/acl-${VERSION_LIBACL}.tar.gz | tar xvz --strip-components=1 --one-top-level=acl -f - -C ${PREFIX}/src
+    cd ${PREFIX}/src/acl
+    ./configure --prefix=${PREFIX}
+    make -j$(nproc) install
+
+    # liblz4
+    VERSION_LIBLZ4=1.9.4
+    git -C ${PREFIX}/src clone --depth 1 --branch v${VERSION_LIBLZ4} https://github.com/lz4/lz4.git
+    cd ${PREFIX}/src/lz4
+    make -j$(nproc) install PREFIX=${PREFIX}
+
+    # libzstd
+    VERSION_LIBZSTD=1.5.4
+    git -C ${PREFIX}/src clone --depth 1 --branch v${VERSION_LIBZSTD} https://github.com/facebook/zstd.git
+    cd ${PREFIX}/src/zstd
+    make -j$(nproc) install PREFIX=${PREFIX}
+
+    # xxHash
+    VERSION_LIBXXHASH=0.8.1
+    git -C ${PREFIX}/src clone --depth 1 --branch v${VERSION_LIBXXHASH} https://github.com/Cyan4973/xxHash.git
+    cd ${PREFIX}/src/xxHash
+    make -j$(nproc) install PREFIX=${PREFIX}
+
+    # openssl
+    VERSION_OPENSSL=1_1_1t
+    git -C ${PREFIX}/src clone --depth 1 --branch OpenSSL_${VERSION_OPENSSL} https://github.com/openssl/openssl.git
+    cd ${PREFIX}/src/openssl
+    ./config --prefix=${PREFIX} --openssldir=${PREFIX}/lib/ssl
+    make -j$(nproc)
+    make -j$(nproc) install
+
+    # libfuse3 requires ninja
+    VERSION_NINJA=1.11.1
+    git -C ${PREFIX}/src clone --depth 1 --branch v${VERSION_NINJA} https://github.com/ninja-build/ninja.git
+    cd ${PREFIX}/src/ninja
+    python3 configure.py --bootstrap
+    install --mode=755 --target-directory=${PREFIX}/bin ninja
+
+    # libfuse3 requires meson >= 0.50; python3.5 support is dropped in meson >= 0.57
+    VERSION_MESON=0.56.2
+    git -C ${PREFIX}/src clone --depth 1 --branch ${VERSION_MESON} https://github.com/mesonbuild/meson.git
+    ln -s ${PREFIX}/src/meson/meson.py ${PREFIX}/bin/meson
+
+    # libfuse3
+    VERSION_LIBFUSE=3.14.0
+    git -C ${PREFIX}/src clone --depth 1 --branch fuse-${VERSION_LIBFUSE} https://github.com/libfuse/libfuse.git
+    cd ${PREFIX}/src/libfuse
+    mkdir build; cd build
+    meson setup --prefix ${PREFIX} --libdir ${PREFIX}/lib ..
+    ninja
+    ninja install
+  EOF
+end
+
 def install_pyenv(boxname)
   return <<-EOF
-    echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile
+    echo 'export PYTHON_CONFIGURE_OPTS="${PYTHON_CONFIGURE_OPTS} --enable-shared"' >> ~/.bash_profile
     echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bash_profile
     echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bash_profile
     . ~/.bash_profile
@@ -160,6 +247,7 @@ end
 def install_pythons(boxname)
   return <<-EOF
     . ~/.bash_profile
+    echo "PYTHON_CONFIGURE_OPTS: ${PYTHON_CONFIGURE_OPTS}"
     pyenv install 3.11.2  # tests, binary build
     pyenv install 3.10.1  # tests
     pyenv install 3.9.1   # tests
@@ -330,6 +418,23 @@ Vagrant.configure(2) do |config|
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("buster64", ".*none.*")
   end
 
+  config.vm.define "stretch64" do |b|
+    b.vm.box = "debian/stretch64"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 1024 + $wmem
+    end
+    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
+    b.vm.provision "install source dependencies", :type => :shell, :privileged => true, :inline => install_source_dependencies("vagrant")
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64", ".*none.*")
+  end
+
   config.vm.define "freebsd64" do |b|
     b.vm.box = "generic/freebsd13"
     b.vm.provider :virtualbox do |v|

docs/changes.rst

@@ -12,8 +12,8 @@ This section provides information about security and corruption issues.
 Change Log 2.x
 ==============
 
-Version 2.0.0b5 (2023-02-27)
-----------------------------
+Version 2.0.0b6 (not released yet)
+----------------------------------
 
 Please note:
 
@@ -108,7 +108,59 @@ Compatibility notes:
 
   Option --filter=... might need an update, if you filter for the status chars
   that were changed.
+- borg is now more strict and disallows giving some options multiple times -
+  if that makes no sense. Highlander options, see #6269. That might make scripts
+  fail now that somehow "worked" before (but maybe didn't work as intended due to
+  the contradicting options).
+
+New features:
+
+- diff: include changes in ctime and mtime, #7248
+- diff: sort JSON output alphabetically
+- diff --content-only: option added to ignore metadata changes
+- import-tar --ignore-zeros: new option to support importing concatenated tars, #7432
+- debug id-hash / parse-obj / format-obj: new debug commands, #7406
 
+Fixes:
+
+- do not retry on permission errors (pointless)
+- transfer: verify chunks we get using assert_id, #7383
+- fix config/cache dir compatibility issues, #7445
+- xattrs: fix namespace processing on FreeBSD, #6997
+- ProgressIndicatorPercent: fix space computation for wide chars, #3027
+- delete: remove --cache-only option, #7440.
+  for deleting the cache only, use: borg rdelete --cache-only
+- borg debug get-obj/put-obj: fixed chunk id
+
+Other changes:
+
+- allow msgpack 1.0.5 also
+- clarify platformdirs requirements, #7393.
+  3.0.0 is only required for macOS due to breaking changes.
+  2.6.0 was the last breaking change for Linux/UNIX.
+- mount: improve mountpoint error msgs, see #7496
+- more Highlander options, #6269
+- Windows: simplify building (just use pip)
+- docs:
+
+  - add installation instructions for Windows
+  - improve --one-file-system help and docs (macOS APFS), #5618 #4876
+  - BORG_KEY_FILE: clarify docs, #7444
+  - installation: add link to OS dependencies, #7356
+  - update FAQ about locale/unicode issues, #6999
+  - improve mount options rendering, #7359
+  - make timestamps in manual pages reproducible.
+- CI / tests / vagrant:
+
+  - added pre-commit for linting purposes, #7476
+  - resolved mode bug and added sleep clause for darwin systems, #7470
+  - "auto" compressor tests: do not assume zlib is better than lz4, #7363
+  - add stretch64 VM with deps built from source
+  - misc. other CI / test fixes and updates
+
+
+Version 2.0.0b5 (2023-02-27)
+----------------------------
 
 New features:
 

docs/development.rst

@@ -179,6 +179,13 @@ virtual env and run::
   pip install -r requirements.d/development.txt
 
 
+This project utilizes pre-commit to format and lint code before it is committed.
+Although pre-commit is installed when running the command above, the pre-commit hooks
+will have to be installed separately. Run this command to install the pre-commit hooks::
+
+  pre-commit install
+
+
 Running the tests
 -----------------
 

docs/usage/general/environment.rst.inc

@@ -104,6 +104,14 @@ General:
             caused EROFS. You will need this to make archives from volume shadow copies
             in WSL1 (Windows Subsystem for Linux 1).
 
+Output formatting:
+    BORG_LIST_FORMAT
+        Giving the default value for ``borg list --format=X``.
+    BORG_RLIST_FORMAT
+        Giving the default value for ``borg rlist --format=X``.
+    BORG_PRUNE_FORMAT
+        Giving the default value for ``borg prune --format=X``.
+
 Some automatic "answerers" (if set, they automatically answer confirmation questions):
     BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes)
         For "Warning: Attempting to access a previously unknown unencrypted repository"
@@ -145,7 +153,16 @@ Directories and files:
         Defaults to ``$BORG_CONFIG_DIR/keys``.
         This directory contains keys for encrypted repositories.
     BORG_KEY_FILE
-        When set, use the given filename as repository key file.
+        When set, use the given path as repository key file. Please note that this is only
+        for rather special applications that externally fully manage the key files:
+
+        - this setting only applies to the keyfile modes (not to the repokey modes).
+        - using a full, absolute path to the key file is recommended.
+        - all directories in the given path must exist.
+        - this setting forces borg to use the key file at the given location.
+        - the key file must either exist (for most commands) or will be created (``borg rcreate``).
+        - you need to give a different path for different repositories.
+        - you need to point to the correct key file matching the repository the command will operate on.
     TMPDIR
         This is where temporary files are stored (might need a lot of temporary space for some
         operations), see tempfile_ for details.

requirements.d/development.txt

@@ -10,3 +10,4 @@ pytest-cov
 pytest-benchmark
 Cython
 twine
+pre-commit

src/borg/archive.py

@@ -885,7 +885,7 @@ def make_parent(path):
                     item_size = item.size
                     if item_size != item_chunks_size:
                         raise BackupError(
-                            "Size inconsistency detected: size {}, chunks size {}".format(item_size, item_chunks_size)
+                            f"Size inconsistency detected: size {item_size}, chunks size {item_chunks_size}"
                         )
                 if has_damaged_chunks:
                     raise BackupError("File has damaged (all-zero) chunks. Try running borg check --repair.")