Remove os.path and add type hints to configparser.py

Author: Armavica

pytensor/configparser.py

@@ -13,6 +13,7 @@
 )
 from functools import wraps
 from io import StringIO
+from pathlib import Path
 
 from pytensor.utils import hash_from_code
 
@@ -22,7 +23,7 @@
 
 class PyTensorConfigWarning(Warning):
     @classmethod
-    def warn(cls, message, stacklevel=0):
+    def warn(cls, message: str, stacklevel: int = 0):
         warnings.warn(message, cls, stacklevel=stacklevel + 3)
 
 
@@ -68,7 +69,123 @@ def __exit__(self, *args):
 class PyTensorConfigParser:
     """Object that holds configuration settings."""
 
-    def __init__(self, flags_dict: dict, pytensor_cfg, pytensor_raw_cfg):
+    # add_basic_configvars
+    floatX: str
+    warn_float64: str
+    pickle_test_value: bool
+    cast_policy: str
+    deterministic: str
+    device: str
+    force_device: bool
+    conv__assert_shape: bool
+    print_global_stats: bool
+    assert_no_cpu_op: str
+    unpickle_function: bool
+    # add_compile_configvars
+    mode: str
+    cxx: str
+    linker: str
+    allow_gc: bool
+    optimizer: str
+    optimizer_verbose: bool
+    on_opt_error: str
+    nocleanup: bool
+    on_unused_import: str
+    gcc__cxxflags: str
+    cmodule__warn_no_version: bool
+    cmodule__remove_gxx_opt: bool
+    cmodule__compilation_warning: bool
+    cmodule__preload_cache: bool
+    cmodule__age_thresh_use: int
+    cmodule__debug: bool
+    compile__wait: int
+    compile__timeout: int
+    ctc__root: str
+    # add_tensor_configvars
+    tensor__cmp_sloppy: int
+    lib__amblibm: bool
+    tensor__insert_inplace_optimizer_validate_nb: int
+    # add_traceback_configvars
+    traceback__limit: int
+    traceback__compile_limit: int
+    # add_experimental_configvars
+    # add_error_and_warning_configvars
+    warn__ignore_bug_before: int
+    exception_verbosity: str
+    # add_testvalue_and_checking_configvars
+    print_test_value: bool
+    compute_test_value: str
+    compute_test_value_opt: str
+    check_input: bool
+    NanGuardMode__nan_is_error: bool
+    NanGuardMode__inf_is_error: bool
+    NanGuardMode__big_is_error: bool
+    NanGuardMode__action: str
+    DebugMode__patience: int
+    DebugMode__check_c: bool
+    DebugMode__check_py: bool
+    DebugMode__check_finite: bool
+    DebugMode__check_strides: int
+    DebugMode__warn_input_not_reused: bool
+    DebugMode__check_preallocated_output: str
+    DebugMode__check_preallocated_output_ndim: int
+    profiling__time_thunks: bool
+    profiling__n_apply: int
+    profiling__n_ops: int
+    profiling__output_line_width: int
+    profiling__min_memory_size: int
+    profiling__min_peak_memory: bool
+    profiling__destination: str
+    profiling__debugprint: bool
+    profiling__ignore_first_call: bool
+    on_shape_error: str
+    # add_multiprocessing_configvars
+    openmp: bool
+    openmp_elemwise_minsize: int
+    # add_optimizer_configvars
+    optimizer_excluding: str
+    optimizer_including: str
+    optimizer_requiring: str
+    optdb__position_cutoff: float
+    optdb__max_use_ratio: float
+    cycle_detection: str
+    check_stack_trace: str
+    metaopt__verbose: int
+    metaopt__optimizer_excluding: str
+    metaopt__optimizer_including: str
+    # add_vm_configvars
+    profile: bool
+    profile_optimizer: bool
+    profile_memory: bool
+    vm__lazy: bool | None
+    # add_deprecated_configvars
+    unittests__rseed: str
+    warn__round: bool
+    # add_scan_configvars
+    scan__allow_gc: bool
+    scan__allow_output_prealloc: bool
+    # add_numba_configvars
+    numba__vectorize_target: str
+    numba__fastmath: bool
+    numba__cache: bool
+    # add_caching_dir_configvars
+    compiledir_format: str
+    base_compiledir: Path
+    compiledir: Path
+    # add_blas_configvars
+    blas__ldflags: str
+    blas__check_openmp: bool
+    # add CUDA (?)
+    cuda__root: Path | None
+    dnn__base_path: Path | None
+    dnn__library_path: Path | None
+
+    def __init__(
+        self,
+        flags_dict: dict,
+        pytensor_cfg: ConfigParser,
+        pytensor_raw_cfg: RawConfigParser,
+    ):
         self._flags_dict = flags_dict
         self._pytensor_cfg = pytensor_cfg
         self._pytensor_raw_cfg = pytensor_raw_cfg
@@ -80,7 +197,7 @@ def __str__(self, print_doc=True):
         self.config_print(buf=sio, print_doc=print_doc)
         return sio.getvalue()
 
-    def config_print(self, buf, print_doc=True):
+    def config_print(self, buf, print_doc: bool = True):
         for cv in self._config_var_dict.values():
             print(cv, file=buf)
             if print_doc:
@@ -108,7 +225,9 @@ def get_config_hash(self):
             )
         )
 
-    def add(self, name, doc, configparam, in_c_key=True):
+    def add(
+        self, name: str, doc: str, configparam: "ConfigParam", in_c_key: bool = True
+    ):
         """Add a new variable to PyTensorConfigParser.
 
         This method performs some of the work of initializing `ConfigParam` instances.
@@ -168,7 +287,7 @@ def add(self, name, doc, configparam, in_c_key=True):
         # the ConfigParam implements __get__/__set__, enabling us to create a property:
         setattr(self.__class__, name, configparam)
 
-    def fetch_val_for_key(self, key, delete_key=False):
+    def fetch_val_for_key(self, key, delete_key: bool = False):
         """Return the overriding config value for a key.
         A successful search returns a string value.
         An unsuccessful search raises a KeyError
@@ -260,9 +379,9 @@ def __init__(
         self._mutable = mutable
         self.is_default = True
         # set by PyTensorConfigParser.add:
-        self.name = None
-        self.doc = None
-        self.in_c_key = None
+        self.name: str = "unnamed"
+        self.doc: str = "undocumented"
+        self.in_c_key: bool
 
         # Note that we do not call `self.filter` on the default value: this
         # will be done automatically in PyTensorConfigParser.add, potentially with a
@@ -288,7 +407,7 @@ def apply(self, value):
             return self._apply(value)
         return value
 
-    def validate(self, value) -> bool | None:
+    def validate(self, value) -> bool:
         """Validates that a parameter values falls into a supported set or range.
 
         Raises
@@ -336,7 +455,7 @@ def __set__(self, cls, val):
 
 class EnumStr(ConfigParam):
     def __init__(
-        self, default: str, options: Sequence[str], validate=None, mutable=True
+        self, default: str, options: Sequence[str], validate=None, mutable: bool = True
     ):
         """Creates a str-based parameter that takes a predefined set of options.
 
@@ -400,7 +519,7 @@ class BoolParam(TypedParam):
     True, 1, "true", "True", "1"
     """
 
-    def __init__(self, default, validate=None, mutable=True):
+    def __init__(self, default, validate=None, mutable: bool = True):
         super().__init__(default, apply=self._apply, validate=validate, mutable=mutable)
 
     def _apply(self, value):
@@ -454,7 +573,9 @@ def _apply(self, val):
         return val
 
 
-def parse_config_string(config_string, issue_warnings=True):
+def parse_config_string(
+    config_string: str, issue_warnings: bool = True
+) -> dict[str, str]:
     """
     Parses a config string (comma-separated key=value components) into a dict.
     """
@@ -480,7 +601,7 @@ def parse_config_string(config_string, issue_warnings=True):
     return config_dict
 
 
-def config_files_from_pytensorrc():
+def config_files_from_pytensorrc() -> list[Path]:
     """
     PYTENSORRC can contain a colon-delimited list of config files, like
 
@@ -489,17 +610,17 @@ def config_files_from_pytensorrc():
     In that case, definitions in files on the right (here, ``~/.pytensorrc``)
     have precedence over those in files on the left.
     """
-    rval = [
-        os.path.expanduser(s)
+    paths = [
+        Path(s).expanduser()
         for s in os.getenv("PYTENSORRC", "~/.pytensorrc").split(os.pathsep)
     ]
     if os.getenv("PYTENSORRC") is None and sys.platform == "win32":
         # to don't need to change the filename and make it open easily
-        rval.append(os.path.expanduser("~/.pytensorrc.txt"))
-    return rval
+        paths.append(Path("~/.pytensorrc.txt").expanduser())
+    return paths
 
 
-def _create_default_config():
+def _create_default_config() -> PyTensorConfigParser:
     # The PYTENSOR_FLAGS environment variable should be a list of comma-separated
     # [section__]option=value entries. If the section part is omitted, there should
     # be only one section that contains the given option.
@@ -509,7 +630,7 @@ def _create_default_config():
     config_files = config_files_from_pytensorrc()
     pytensor_cfg = ConfigParser(
         {
-            "USER": os.getenv("USER", os.path.split(os.path.expanduser("~"))[-1]),
+            "USER": os.getenv("USER", Path("~").expanduser().name),
             "LSCRATCH": os.getenv("LSCRATCH", ""),
             "TMPDIR": os.getenv("TMPDIR", ""),
             "TEMP": os.getenv("TEMP", ""),

pytensor/link/c/cmodule.py

@@ -23,6 +23,7 @@
 from collections.abc import Callable
 from contextlib import AbstractContextManager, nullcontext
 from io import BytesIO, StringIO
+from pathlib import Path
 from typing import TYPE_CHECKING, Protocol, cast
 
 import numpy as np
@@ -688,7 +689,7 @@ class ModuleCache:
 
     """
 
-    dirname: str = ""
+    dirname: Path
     """
     The working directory that is managed by this interface.
 
@@ -725,8 +726,13 @@ class ModuleCache:
 
     """
 
-    def __init__(self, dirname, check_for_broken_eq=True, do_refresh=True):
-        self.dirname = dirname
+    def __init__(
+        self,
+        dirname: Path | str,
+        check_for_broken_eq: bool = True,
+        do_refresh: bool = True,
+    ):
+        self.dirname = Path(dirname)
         self.module_from_name = dict(self.module_from_name)
         self.entry_from_key = dict(self.entry_from_key)
         self.module_hash_to_key_data = dict(self.module_hash_to_key_data)
@@ -1637,12 +1643,12 @@ def _rmtree(
 _module_cache: ModuleCache | None = None
 
 
-def get_module_cache(dirname: str, init_args=None) -> ModuleCache:
+def get_module_cache(dirname: Path | str, init_args=None) -> ModuleCache:
     """Create a new module_cache.
 
     Parameters
     ----------
-    dirname
+    dirname : Path | str
         The name of the directory used by the cache.
     init_args
         Keyword arguments passed to the `ModuleCache` constructor.

pytensor/link/c/cutils.py

@@ -1,6 +1,7 @@
 import errno
 import os
 import sys
+from pathlib import Path
 
 from pytensor.compile.compilelock import lock_ctx
 from pytensor.configdefaults import config
@@ -87,8 +88,8 @@ def compile_cutils():
     # for the same reason. Note that these 5 lines may seem redundant (they are
     # repeated in compile_str()) but if another cutils_ext does exist then it
     # will be imported and compile_str won't get called at all.
-    sys.path.insert(0, config.compiledir)
-    location = os.path.join(config.compiledir, "cutils_ext")
+    sys.path.insert(0, str(config.compiledir))
+    location = os.path.join(str(config.compiledir), "cutils_ext")
     if not os.path.exists(location):
         try:
             os.mkdir(location)
@@ -115,5 +116,5 @@ def compile_cutils():
                 compile_cutils()
                 from cutils_ext.cutils_ext import *  # noqa
 finally:
-    if sys.path[0] == config.compiledir:
+    if config.compiledir.resolve() == Path(sys.path[0]).resolve():
         del sys.path[0]

tests/test_config.py

@@ -138,7 +138,7 @@ def test_deviceparam(self):
             cp._apply("gpu123")
         with pytest.raises(ValueError, match='Valid options start with one of "cpu".'):
             cp._apply("notadevice")
-        assert str(cp) == "None (cpu)"
+        assert str(cp) == "unnamed (cpu)"
 
 
 def test_config_context():