implement plan

Author: ThomasWaldmann

src/borg/archive.py

@@ -2,6 +2,7 @@
 import errno
 import json
 import os
+import posixpath
 import stat
 import sys
 import time
@@ -1243,8 +1244,8 @@ def __init__(
     @contextmanager
     def create_helper(self, path, st, status=None, hardlinkable=True, strip_prefix=None):
         if strip_prefix is not None:
-            assert not path.endswith(os.sep)
-            if strip_prefix.startswith(path + os.sep):
+            assert not path.endswith("/")
+            if strip_prefix.startswith(path + "/"):
                 # still on a directory level that shall be stripped - do not create an item for this!
                 yield None, "x", False, None
                 return
@@ -1547,7 +1548,7 @@ def s_to_ns(s):
 
             # if the tar has names starting with "./", normalize them like borg create also does.
             # ./dir/file must become dir/file in the borg archive.
-            normalized_path = os.path.normpath(tarinfo.name)
+            normalized_path = posixpath.normpath(tarinfo.name)
             item = Item(
                 path=make_path_safe(normalized_path),
                 mode=tarinfo.mode | type,
@@ -1608,7 +1609,7 @@ def process_symlink(self, *, tarinfo, status, type):
     def process_hardlink(self, *, tarinfo, status, type):
         with self.create_helper(tarinfo, status, type) as (item, status):
             # create a not hardlinked borg item, reusing the chunks, see HardLinkManager.__doc__
-            normalized_path = os.path.normpath(tarinfo.linkname)
+            normalized_path = posixpath.normpath(tarinfo.linkname)
             safe_path = make_path_safe(normalized_path)
             chunks = self.hlm.retrieve(safe_path)
             if chunks is not None:

src/borg/archiver/create_cmd.py

@@ -3,6 +3,7 @@
 import argparse
 import logging
 import os
+import posixpath
 import stat
 import subprocess
 import time
@@ -16,11 +17,11 @@
 from ..cache import Cache
 from ..constants import *  # NOQA
 from ..compress import CompressionSpec
-from ..helpers import comment_validator, ChunkerParams, PathSpec
+from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec
 from ..helpers import archivename_validator, FilesCacheMode
 from ..helpers import eval_escapes
 from ..helpers import timestamp, archive_ts_now
-from ..helpers import get_cache_dir, os_stat, get_strip_prefix
+from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
 from ..helpers import dir_is_tagged
 from ..helpers import log_multi
 from ..helpers import basic_json_data, json_print
@@ -106,8 +107,9 @@ def create_inner(archive, cache, fso):
                     pipe_bin = sys.stdin.buffer
                 pipe = TextIOWrapper(pipe_bin, errors="surrogateescape")
                 for path in iter_separated(pipe, paths_sep):
+                    path = slashify(path)
                     strip_prefix = get_strip_prefix(path)
-                    path = os.path.normpath(path)
+                    path = posixpath.normpath(path)
                     try:
                         with backup_io("stat"):
                             st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
@@ -160,7 +162,7 @@ def create_inner(archive, cache, fso):
                         continue
 
                     strip_prefix = get_strip_prefix(path)
-                    path = os.path.normpath(path)
+                    path = posixpath.normpath(path)
                     try:
                         with backup_io("stat"):
                             st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
@@ -489,7 +491,7 @@ def _rec_walk(
                                             path=path, fd=child_fd, st=st, strip_prefix=strip_prefix
                                         )
                                     for tag_name in tag_names:
-                                        tag_path = os.path.join(path, tag_name)
+                                        tag_path = posixpath.join(path, tag_name)
                                         self._rec_walk(
                                             path=tag_path,
                                             parent_fd=child_fd,
@@ -523,7 +525,7 @@ def _rec_walk(
                         with backup_io("scandir"):
                             entries = helpers.scandir_inorder(path=path, fd=child_fd)
                         for dirent in entries:
-                            normpath = os.path.normpath(os.path.join(path, dirent.name))
+                            normpath = posixpath.normpath(posixpath.join(path, dirent.name))
                             self._rec_walk(
                                 path=normpath,
                                 parent_fd=child_fd,
@@ -962,5 +964,5 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
 
         subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name")
         subparser.add_argument(
-            "paths", metavar="PATH", nargs="*", type=PathSpec, action="extend", help="paths to archive"
+            "paths", metavar="PATH", nargs="*", type=FilesystemPathSpec, action="extend", help="paths to archive"
         )

src/borg/archiver/extract_cmd.py

@@ -1,7 +1,6 @@
 import sys
 import argparse
 import logging
-import os
 import stat
 
 from ._common import with_repository, with_archive
@@ -60,7 +59,7 @@ def do_extract(self, args, repository, manifest, archive):
         for item in archive.iter_items():
             orig_path = item.path
             if strip_components:
-                stripped_path = os.sep.join(orig_path.split(os.sep)[strip_components:])
+                stripped_path = "/".join(orig_path.split("/")[strip_components:])
                 if not stripped_path:
                     continue
                 item.path = stripped_path

src/borg/helpers/__init__.py

@@ -20,15 +20,23 @@
 from .fs import ensure_dir, join_base_dir, get_socket_filename
 from .fs import get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get_config_dir, get_runtime_dir
 from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder
-from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount
+from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify
 from .fs import O_, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
 from .fs import HardLinkManager
 from .misc import sysinfo, log_multi, consume
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
 from .parseformat import bin_to_hex, hex_to_bin, safe_encode, safe_decode
 from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd
 from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval
-from .parseformat import PathSpec, SortBySpec, ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper
+from .parseformat import (
+    PathSpec,
+    FilesystemPathSpec,
+    SortBySpec,
+    ChunkerParams,
+    FilesCacheMode,
+    partial_format,
+    DatetimeWrapper,
+)
 from .parseformat import format_file_size, parse_file_size, FileSize
 from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator
 from .parseformat import format_line, replace_placeholders, PlaceholderError, relative_time_marker_validator

src/borg/helpers/fs.py

@@ -249,13 +249,38 @@ def make_path_safe(path):
     For reasons of security, a ValueError is raised should
     `path` contain any '..' elements.
     """
+    if "\\.." in path or "..\\" in path:
+        raise ValueError(f"unexpected '..' element in path {path!r}")
+
+    path = percentify(path)
+
     path = path.lstrip("/")
     if path.startswith("../") or "/../" in path or path.endswith("/..") or path == "..":
         raise ValueError(f"unexpected '..' element in path {path!r}")
     path = posixpath.normpath(path)
     return path
 
 
+def slashify(path):
+    """
+    Replace backslashes with forward slashes if running on Windows.
+
+    Use case: we always want to use forward slashes, even on Windows.
+    """
+    return path.replace("\\", "/") if is_win32 else path
+
+
+def percentify(path):
+    """
+    Replace backslashes with percent signs if running on Windows.
+
+    Use case: if an archived path contains backslashes (which is not a path separator on POSIX
+    and could appear as a normal character in POSIX paths), we need to replace them with percent
+    signs to make the path usable on Windows.
+    """
+    return path.replace("\\", "%") if is_win32 else path
+
+
 def get_strip_prefix(path):
     # similar to how rsync does it, we allow users to give paths like:
     # /this/gets/stripped/./this/is/kept
@@ -265,7 +290,7 @@ def get_strip_prefix(path):
     pos = path.find("/./")  # detect slashdot hack
     if pos > 0:
         # found a prefix to strip! make sure it ends with one "/"!
-        return os.path.normpath(path[:pos]) + os.sep
+        return posixpath.normpath(path[:pos]) + "/"
     else:
         # no or empty prefix, nothing to strip!
         return None
@@ -276,15 +301,14 @@ def get_strip_prefix(path):
 
 def remove_dotdot_prefixes(path):
     """
-    Remove '../'s at the beginning of `path`. Additionally,
-    the path is made relative.
+    Remove '../'s at the beginning of `path`. Additionally, the path is made relative.
 
-    `path` is expected to be normalized already (e.g. via `os.path.normpath()`).
+    `path` is expected to be normalized already (e.g. via `posixpath.normpath()`).
     """
+    assert "\\" not in path
     if is_win32:
         if len(path) > 1 and path[1] == ":":
             path = path.replace(":", "", 1)
-        path = path.replace("\\", "/")
 
     path = path.lstrip("/")
     path = _dotdot_re.sub("", path)

src/borg/helpers/parseformat.py

@@ -23,12 +23,13 @@
 logger = create_logger()
 
 from .errors import Error
-from .fs import get_keys_dir, make_path_safe
+from .fs import get_keys_dir, make_path_safe, slashify
 from .msgpack import Timestamp
 from .time import OutputTimestamp, format_time, safe_timestamp
 from .. import __version__ as borg_version
 from .. import __version_tuple__ as borg_version_tuple
 from ..constants import *  # NOQA
+from ..platformflags import is_win32
 
 if TYPE_CHECKING:
     from ..item import ItemDiff
@@ -335,6 +336,12 @@ def PathSpec(text):
     return text
 
 
+def FilesystemPathSpec(text):
+    if not text:
+        raise argparse.ArgumentTypeError("Empty strings are not accepted as paths.")
+    return slashify(text)
+
+
 def SortBySpec(text):
     from ..manifest import AI_HUMAN_SORT_KEYS
 
@@ -558,7 +565,8 @@ def _parse(self, text):
         m = self.local_re.match(text)
         if m:
             self.proto = "file"
-            self.path = os.path.abspath(os.path.normpath(m.group("path")))
+            path = m.group("path")
+            self.path = slashify(os.path.abspath(path)) if is_win32 else os.path.abspath(path)
             return True
         return False
 

src/borg/item.pyx

@@ -7,7 +7,7 @@ from cpython.bytes cimport PyBytes_AsStringAndSize
 from .constants import ITEM_KEYS, ARCHIVE_KEYS
 from .helpers import StableDict
 from .helpers import format_file_size
-from .helpers.fs import assert_sanitized_path, to_sanitized_path
+from .helpers.fs import assert_sanitized_path, to_sanitized_path, percentify, slashify
 from .helpers.msgpack import timestamp_to_int, int_to_timestamp, Timestamp
 from .helpers.time import OutputTimestamp, safe_timestamp
 
@@ -265,7 +265,7 @@ cdef class Item(PropDict):
 
     path = PropDictProperty(str, 'surrogate-escaped str', encode=assert_sanitized_path, decode=to_sanitized_path)
     source = PropDictProperty(str, 'surrogate-escaped str')  # legacy borg 1.x. borg 2: see .target
-    target = PropDictProperty(str, 'surrogate-escaped str')
+    target = PropDictProperty(str, 'surrogate-escaped str', encode=slashify, decode=percentify)
     user = PropDictProperty(str, 'surrogate-escaped str')
     group = PropDictProperty(str, 'surrogate-escaped str')
 

src/borg/patterns.py

@@ -1,6 +1,6 @@
 import argparse
 import fnmatch
-import os.path
+import posixpath
 import re
 import sys
 import unicodedata
@@ -142,7 +142,7 @@ def match(self, path):
         in self.fallback is returned (defaults to None).
 
         """
-        path = normalize_path(path).lstrip(os.path.sep)
+        path = normalize_path(path).lstrip("/")
         # do a fast lookup for full path matches (note: we do not count such matches):
         non_existent = object()
         value = self._path_full_patterns.get(path, non_existent)
@@ -215,7 +215,7 @@ class PathFullPattern(PatternBase):
     PREFIX = "pf"
 
     def _prepare(self, pattern):
-        self.pattern = os.path.normpath(pattern).lstrip(os.path.sep)  # sep at beginning is removed
+        self.pattern = posixpath.normpath(pattern).lstrip("/")  # / at beginning is removed
 
     def _match(self, path):
         return path == self.pattern
@@ -236,12 +236,10 @@ class PathPrefixPattern(PatternBase):
     PREFIX = "pp"
 
     def _prepare(self, pattern):
-        sep = os.path.sep
-
-        self.pattern = (os.path.normpath(pattern).rstrip(sep) + sep).lstrip(sep)  # sep at beginning is removed
+        self.pattern = (posixpath.normpath(pattern).rstrip("/") + "/").lstrip("/")  # / at beginning is removed
 
     def _match(self, path):
-        return (path + os.path.sep).startswith(self.pattern)
+        return (path + "/").startswith(self.pattern)
 
 
 class FnmatchPattern(PatternBase):
@@ -252,19 +250,19 @@ class FnmatchPattern(PatternBase):
     PREFIX = "fm"
 
     def _prepare(self, pattern):
-        if pattern.endswith(os.path.sep):
-            pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + "*" + os.path.sep
+        if pattern.endswith("/"):
+            pattern = posixpath.normpath(pattern).rstrip("/") + "/*/"
         else:
-            pattern = os.path.normpath(pattern) + os.path.sep + "*"
+            pattern = posixpath.normpath(pattern) + "/*"
 
-        self.pattern = pattern.lstrip(os.path.sep)  # sep at beginning is removed
+        self.pattern = pattern.lstrip("/")  # / at beginning is removed
 
         # fnmatch and re.match both cache compiled regular expressions.
         # Nevertheless, this is about 10 times faster.
         self.regex = re.compile(fnmatch.translate(self.pattern))
 
     def _match(self, path):
-        return self.regex.match(path + os.path.sep) is not None
+        return self.regex.match(path + "/") is not None
 
 
 class ShellPattern(PatternBase):
@@ -275,18 +273,16 @@ class ShellPattern(PatternBase):
     PREFIX = "sh"
 
     def _prepare(self, pattern):
-        sep = os.path.sep
-
-        if pattern.endswith(sep):
-            pattern = os.path.normpath(pattern).rstrip(sep) + sep + "**" + sep + "*" + sep
+        if pattern.endswith("/"):
+            pattern = posixpath.normpath(pattern).rstrip("/") + "/**/*/"
         else:
-            pattern = os.path.normpath(pattern) + sep + "**" + sep + "*"
+            pattern = posixpath.normpath(pattern) + "/**/*"
 
-        self.pattern = pattern.lstrip(sep)  # sep at beginning is removed
+        self.pattern = pattern.lstrip("/")  # / at beginning is removed
         self.regex = re.compile(shellpattern.translate(self.pattern))
 
     def _match(self, path):
-        return self.regex.match(path + os.path.sep) is not None
+        return self.regex.match(path + "/") is not None
 
 
 class RegexPattern(PatternBase):
@@ -295,14 +291,11 @@ class RegexPattern(PatternBase):
     PREFIX = "re"
 
     def _prepare(self, pattern):
-        self.pattern = pattern  # sep at beginning is NOT removed
+        self.pattern = pattern  # / at beginning is NOT removed
         self.regex = re.compile(pattern)
 
     def _match(self, path):
-        # Normalize path separators
-        if os.path.sep != "/":
-            path = path.replace(os.path.sep, "/")
-
+        assert "\\" not in path
         return self.regex.search(path) is not None