Guard against non-ascii characters in shell commands (#893)

* subclass ArgumentParser so that non-ascii characters are guarded against

* fix super call

* format

* Update wkcuber/wkcuber/_internal/utils.py

Co-authored-by: Florian M <[email protected]>

* add comment

* also allow --allow-non-ascii even if no ascii chars exist in argv

---------

Co-authored-by: Florian M <[email protected]>

Author: philippotto

wkcuber/wkcuber/_internal/utils.py

@@ -1,9 +1,12 @@
+import sys
+import re
+from argparse import ArgumentParser, Namespace
 from collections import namedtuple
 from glob import iglob
 from logging import getLogger
 from math import ceil, floor
 from os import environ
-from typing import Generator, Sequence, Tuple, Union
+from typing import Generator, Sequence, Tuple, Union, Optional
 
 import numpy as np
 import wkw
@@ -65,6 +68,34 @@ def parse_padding(padding_str: str) -> Tuple[int, ...]:
         raise argparse.ArgumentTypeError("The padding could not be parsed") from e
 
 
+class AsciiArgumentParser(ArgumentParser):
+    """
+    This class is a lightweight wrapper around ArgumentParser and checks
+    that the shell command doesn't contain non-ascii characters since these
+    can easily disturb the argument parsing.
+
+    If non-ascii characters are found, an error is raised. This behavior
+    can be disabled by passing --allow-non-ascii on the command line.
+    """
+
+    IGNORE_FLAG = "--allow-non-ascii"
+
+    def parse_args(self, args: Optional[Sequence[str]] = None, namespace: Optional[Namespace] = None) -> Any:  # type: ignore
+        if args is None:
+            args = sys.argv[1:]
+
+        # Search for non-ascii codes (ASCII itself spans 0-127 or 0x00 to 0x7F).
+        results = re.findall(r"[^\x00-\x7F]", " ".join(args))
+        if len(results) > 0:
+            if not self.IGNORE_FLAG in args:
+                raise ValueError(
+                    f"The shell command contains non-ascii characters. Please remove them to avoid unintended effects. If you need them, pass {self.IGNORE_FLAG} to suppress this error. The special characters are: {results}"
+                )
+
+        args = [el for el in args if el != self.IGNORE_FLAG]
+        return super().parse_args(args, namespace)
+
+
 def open_knossos(info: KnossosDatasetInfo) -> KnossosDataset:
     return KnossosDataset.open(info.dataset_path, np.dtype(info.dtype))
 

wkcuber/wkcuber/check_equality.py

@@ -7,6 +7,7 @@
 from webknossos import Dataset, View
 
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_distribution_flags,
     add_verbose_flag,
     parse_path,
@@ -17,7 +18,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path", help="Path to input WKW dataset", type=parse_path

wkcuber/wkcuber/compress.py

@@ -7,6 +7,7 @@
 from webknossos.utils import rmtree
 
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_distribution_flags,
     add_verbose_flag,
     parse_path,
@@ -18,7 +19,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path",

wkcuber/wkcuber/convert_knossos.py

@@ -10,6 +10,7 @@
 
 from ._internal.knossos import CUBE_EDGE_LEN
 from ._internal.utils import (
+    AsciiArgumentParser,
     KnossosDatasetInfo,
     add_data_format_flags,
     add_distribution_flags,
@@ -24,7 +25,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path",

wkcuber/wkcuber/convert_nifti.py

@@ -11,6 +11,7 @@
 from webknossos.utils import time_start, time_stop
 
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_data_format_flags,
     add_verbose_flag,
     add_voxel_size_flag,
@@ -23,7 +24,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path",

wkcuber/wkcuber/convert_raw.py

@@ -15,6 +15,7 @@
 )
 
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_data_format_flags,
     add_distribution_flags,
     add_interpolation_flag,
@@ -47,7 +48,7 @@ def parse_flip_axes(flip_axes: str) -> Tuple[int, ...]:
 
 
 def create_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path",

wkcuber/wkcuber/convert_zarr.py

@@ -21,6 +21,7 @@
 from webknossos.utils import get_executor_for_args, wait_and_ensure_success
 
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_data_format_flags,
     add_distribution_flags,
     add_interpolation_flag,
@@ -46,7 +47,7 @@ def parse_flip_axes(flip_axes: str) -> Tuple[int, ...]:
 
 
 def create_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path",

wkcuber/wkcuber/converter.py

@@ -9,6 +9,7 @@
 
 from ._internal.image_readers import image_reader
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_data_format_flags,
     add_verbose_flag,
     add_voxel_size_flag,
@@ -32,7 +33,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser(
+    parser = AsciiArgumentParser(
         epilog="If you want to pass more specific config values, please use the individual converters. See the readme for a complete overview."
     )
 

wkcuber/wkcuber/cubing.py

@@ -29,6 +29,7 @@
 
 from ._internal.image_readers import image_reader
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_batch_size_flag,
     add_data_format_flags,
     add_distribution_flags,
@@ -46,7 +47,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "source_path", help="Directory containing the input images.", type=Path

wkcuber/wkcuber/downsampling.py

@@ -6,6 +6,7 @@
 from webknossos.utils import get_executor_for_args
 
 from ._internal.utils import (
+    AsciiArgumentParser,
     add_distribution_flags,
     add_interpolation_flag,
     add_isotropic_flag,
@@ -18,7 +19,7 @@
 
 
 def create_parser() -> ArgumentParser:
-    parser = ArgumentParser()
+    parser = AsciiArgumentParser()
 
     parser.add_argument(
         "path", help="Directory containing the dataset.", type=parse_path