chore: fix lints

Author: tanloong

pyproject.toml

@@ -75,7 +75,11 @@ select = [
     "SIM", # flake8-simplify
     "I", # isort
 ]
-ignore = ["E501", "B011"]
+ignore = [
+    "E501",
+    "B011",
+    "UP038", # 'isinstance(x, int | float)' causes slower code
+]
 
 [tool.ruff.lint.flake8-type-checking]
 # Add quotes around type annotations, if doing so would allow

src/neosca/ns_envar.py

@@ -3,7 +3,6 @@
 import logging
 import os
 import os.path as os_path
-from typing import List, Optional
 
 from neosca.ns_platform_info import IS_LINUX, IS_MAC, IS_WINDOWS
 from neosca.ns_print import color_print
@@ -13,7 +12,7 @@
 STANFORD_TREGEX_HOME = "STANFORD_TREGEX_HOME"
 
 
-def _setenv_windows(env_var: str, paths: List[str], is_refresh: bool = False) -> None:
+def _setenv_windows(env_var: str, paths: list[str], is_refresh: bool = False) -> None:
     import ctypes  # Allows interface with low-level C API's
     import winreg  # Allows access to the windows registry
 
@@ -47,7 +46,7 @@ def _setenv_windows(env_var: str, paths: List[str], is_refresh: bool = False) ->
         )
 
 
-def _setenv_unix(env_var: str, paths: List[str], is_refresh: bool = False) -> None:
+def _setenv_unix(env_var: str, paths: list[str], is_refresh: bool = False) -> None:
     shell = os.environ.get("SHELL")
     if shell is None:
         logging.warning(
@@ -101,7 +100,7 @@ def _setenv_unix(env_var: str, paths: List[str], is_refresh: bool = False) -> No
                 f.write("\n".join(configs))
 
 
-def setenv(envar: str, paths: List[str], is_override: bool = False, is_quiet: bool = False) -> None:
+def setenv(envar: str, paths: list[str], is_override: bool = False, is_quiet: bool = False) -> None:
     assert any((IS_WINDOWS, IS_MAC, IS_LINUX))
     if IS_WINDOWS:
         _setenv_windows(envar, paths, is_override)
@@ -116,6 +115,6 @@ def setenv(envar: str, paths: List[str], is_override: bool = False, is_quiet: bo
         )
 
 
-def get_dir_frm_env(envar: str) -> Optional[str]:
+def get_dir_frm_env(envar: str) -> str | None:
     directory = os.getenv(envar, "")
     return directory if os_path.isdir(directory) else None

src/neosca/ns_io.py

@@ -9,9 +9,10 @@
 import pickle
 import sys
 import zipfile
+from collections.abc import Generator, Iterable, Sequence
 from os import PathLike
 from pathlib import Path
-from typing import Any, Dict, Generator, Iterable, List, Sequence, Tuple, Union
+from typing import Any
 from xml.etree.ElementTree import XML, fromstring
 
 from charset_normalizer import detect
@@ -32,7 +33,7 @@ def __new__(cls, name, bases, dict_):
 class Ns_IO(metaclass=Ns_IO_Meta):
     # Type checker does not detect definition in Ns_IO_Meta, so declare here to
     # silence the "access unknown member warning"
-    SUPPORTED_EXTENSIONS: Tuple[str, ...] = tuple()
+    SUPPORTED_EXTENSIONS: tuple[str, ...] = tuple()
 
     DOCX_NAMESPACE = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
     DOCX_PARA = DOCX_NAMESPACE + "p"
@@ -95,7 +96,7 @@ def read_odt(cls, path: str) -> str:
         return "\n".join("".join(node.itertext()) for node in paragraphs)
 
     @classmethod
-    def suffix(cls, file_path: Union[str, PathLike], *, strip_dot: bool = False) -> str:
+    def suffix(cls, file_path: str | PathLike, *, strip_dot: bool = False) -> str:
         """
         >>> suffix('my/library/setup.py')
         .py
@@ -112,14 +113,14 @@ def suffix(cls, file_path: Union[str, PathLike], *, strip_dot: bool = False) ->
         return extension
 
     @classmethod
-    def supports(cls, file_path: Union[str, PathLike]) -> bool:
+    def supports(cls, file_path: str | PathLike) -> bool:
         # Can instead use hasattr(f"read_{extension}").
         # The SUPPORTED_EXTENSIONS is required by ns_main_cli:74 to list
         #  supported extensions to users.
         return cls.suffix(file_path, strip_dot=True) in cls.SUPPORTED_EXTENSIONS
 
     @classmethod
-    def not_supports(cls, file_path: Union[str, PathLike]) -> bool:
+    def not_supports(cls, file_path: str | PathLike) -> bool:
         return not cls.supports(file_path)
 
     @classmethod
@@ -152,16 +153,14 @@ def is_writable(cls, filename: str) -> Ns_Procedure_Result:
             return True, None
 
     @classmethod
-    def _check_file_extension(
-        cls, file_path: Union[str, PathLike], valid_extensions: Union[str, Tuple[str, ...]]
-    ):
+    def _check_file_extension(cls, file_path: str | PathLike, valid_extensions: str | tuple[str, ...]):
         if not os_path.isfile(file_path):
             raise FileNotFoundError(f"File {file_path} does not exist")
         if not str(file_path).endswith(valid_extensions):
             raise ValueError(f"{file_path} does not have a valid extension")
 
     @classmethod
-    def load_pickle_lzma(cls, file_path: Union[str, PathLike]) -> Any:
+    def load_pickle_lzma(cls, file_path: str | PathLike) -> Any:
         cls._check_file_extension(file_path, (".pickle.lzma", ".pkl.lzma"))
 
         with open(file_path, "rb") as f:
@@ -171,30 +170,30 @@ def load_pickle_lzma(cls, file_path: Union[str, PathLike]) -> Any:
         return pickle.loads(data_pickle)
 
     @classmethod
-    def load_pickle(cls, file_path: Union[str, PathLike]) -> Any:
+    def load_pickle(cls, file_path: str | PathLike) -> Any:
         cls._check_file_extension(file_path, (".pickle", ".pkl"))
 
         with open(file_path, "rb") as f:
             data_pickle = f.read()
         return pickle.loads(data_pickle)
 
     @classmethod
-    def load_lzma(cls, file_path: Union[str, PathLike]) -> bytes:
+    def load_lzma(cls, file_path: str | PathLike) -> bytes:
         cls._check_file_extension(file_path, ".lzma")
 
         with open(file_path, "rb") as f:
             data_lzma = f.read()
         return lzma.decompress(data_lzma)
 
     @classmethod
-    def load_json(cls, file_path: Union[str, PathLike]) -> Any:
+    def load_json(cls, file_path: str | PathLike) -> Any:
         cls._check_file_extension(file_path, ".json")
 
         with open(file_path, "rb") as f:
             return json.load(f)
 
     @classmethod
-    def dump_json(cls, data: Any, path: Union[str, PathLike]) -> None:
+    def dump_json(cls, data: Any, path: str | PathLike) -> None:
         try:
             with open(path, "w") as f:
                 json.dump(data, f, ensure_ascii=False, indent=2)
@@ -203,7 +202,7 @@ def dump_json(cls, data: Any, path: Union[str, PathLike]) -> None:
             cls.dump_json(data, path)
 
     @classmethod
-    def dump_bytes(cls, data: bytes, path: Union[str, PathLike]) -> None:
+    def dump_bytes(cls, data: bytes, path: str | PathLike) -> None:
         try:
             with open(path, "wb") as f:
                 f.write(data)
@@ -212,7 +211,7 @@ def dump_bytes(cls, data: bytes, path: Union[str, PathLike]) -> None:
             cls.dump_bytes(data, path)
 
     @classmethod
-    def get_verified_ifile_list(cls, ifile_list: Iterable[str]) -> List[str]:
+    def get_verified_ifile_list(cls, ifile_list: Iterable[str]) -> list[str]:
         verified_ifile_list = []
         for path in ifile_list:
             # File path
@@ -245,10 +244,10 @@ def get_verified_ifile_list(cls, ifile_list: Iterable[str]) -> List[str]:
         return verified_ifile_list
 
     @classmethod
-    def get_verified_subfiles_list(cls, subfiles_list: List[List[str]]) -> List[List[str]]:
+    def get_verified_subfiles_list(cls, subfiles_list: list[list[str]]) -> list[list[str]]:
         verified_subfiles_list = []
         for subfiles in subfiles_list:
-            verified_subfiles: List[str] = cls.get_verified_ifile_list(subfiles)
+            verified_subfiles: list[str] = cls.get_verified_ifile_list(subfiles)
             if len(verified_subfiles) == 1:
                 logging.critical(
                     f"Only 1 subfile provided: ({verified_subfiles.pop()}). There should be 2"
@@ -271,15 +270,15 @@ def ensure_unique_filestem(cls, stem: str, existing_stems: Sequence[str]) -> str
 class Ns_Cache:
     CACHE_EXTENSION = ".pickle.lzma"
     # fpath_cname: { "/absolute/path/to/foo.txt": "foo.pickle.lzma", ... }
-    fpath_cname: Dict[str, str] = (
+    fpath_cname: dict[str, str] = (
         Ns_IO.load_json(CACHE_INFO_PATH)
         if CACHE_INFO_PATH.exists() and os_path.getsize(CACHE_INFO_PATH) > 0
         else {}
     )
     info_changed: bool = False
 
     @classmethod
-    def get_cache_path(cls, file_path: str) -> Tuple[str, bool]:
+    def get_cache_path(cls, file_path: str) -> tuple[str, bool]:
         """
         return (cache_path, available: whether the cache is usable)
         """
@@ -306,7 +305,7 @@ def get_cache_path(cls, file_path: str) -> Tuple[str, bool]:
         return cache_path, True
 
     @classmethod
-    def _size_fmt(cls, filesize: Union[int, float], suffix: str = "B") -> str:
+    def _size_fmt(cls, filesize: int | float, suffix: str = "B") -> str:
         # https://github.com/gaogaotiantian/viztracer/blob/3ecd46aa0e70df7dd78f720a2660d6da211c4a51/src/viztracer/util.py#L12
         for unit in ("", "Ki", "Mi", "Gi"):
             if abs(filesize) < 1024.0:
@@ -315,7 +314,7 @@ def _size_fmt(cls, filesize: Union[int, float], suffix: str = "B") -> str:
         return f"{filesize:.1f} {'Ti'}{suffix}"
 
     @classmethod
-    def yield_cname_cpath_csize_fpath(cls) -> Generator[Tuple[str, str, str, str], None, None]:
+    def yield_cname_cpath_csize_fpath(cls) -> Generator[tuple[str, str, str, str], None, None]:
         for file_path, cache_name in Ns_Cache.fpath_cname.items():
             cache_path = Ns_Cache._name2path(cache_name)
             if not os_path.exists(cache_path):

src/neosca/ns_lca/ns_lca.py

@@ -4,7 +4,7 @@
 import os
 import os.path as os_path
 import sys
-from typing import Dict, List, Literal, Optional, Tuple, Union
+from typing import Literal
 
 from neosca.ns_io import Ns_Cache, Ns_IO
 from neosca.ns_lca.ns_lca_counter import Ns_LCA_Counter
@@ -50,16 +50,14 @@ def __init__(
         self.is_save_matches = is_save_matches
         self.is_save_values = is_save_values
 
-        self.counters: List[Ns_LCA_Counter] = []
+        self.counters: list[Ns_LCA_Counter] = []
 
-    def get_lempos_frm_text(
-        self, text: str, /, cache_path: Optional[str] = None
-    ) -> Tuple[Tuple[str, str], ...]:
+    def get_lempos_frm_text(self, text: str, /, cache_path: str | None = None) -> tuple[tuple[str, str], ...]:
         from neosca.ns_nlp import Ns_NLP_Stanza
 
         return Ns_NLP_Stanza.get_lemma_and_pos(text, tagset=self.tagset, cache_path=cache_path)
 
-    def get_lempos_frm_file(self, file_path: str, /) -> Tuple[Tuple[str, str], ...]:
+    def get_lempos_frm_file(self, file_path: str, /) -> tuple[tuple[str, str], ...]:
         from neosca.ns_nlp import Ns_NLP_Stanza
 
         cache_path, is_cache_available = Ns_Cache.get_cache_path(file_path)
@@ -73,7 +71,7 @@ def get_lempos_frm_file(self, file_path: str, /) -> Tuple[Tuple[str, str], ...]:
         text = Ns_IO.load_file(file_path)
 
         if not self.is_cache:
-            cache_path: Optional[str] = None  # type: ignore
+            cache_path: str | None = None  # type: ignore
 
         try:
             return self.get_lempos_frm_text(text)
@@ -107,7 +105,7 @@ def run_on_text(self, text: str, *, file_path: str = "cli_text", clear: bool = T
         if self.is_save_values:
             self.dump_values()
 
-    def run_on_file_or_subfiles(self, file_or_subfiles: Union[str, List[str]]) -> Ns_LCA_Counter:
+    def run_on_file_or_subfiles(self, file_or_subfiles: str | list[str]) -> Ns_LCA_Counter:
         if isinstance(file_or_subfiles, str):
             file_path = file_or_subfiles
             lempos_tuples = self.get_lempos_frm_file(file_path)
@@ -126,7 +124,7 @@ def run_on_file_or_subfiles(self, file_or_subfiles: Union[str, List[str]]) -> Ns
         return counter
 
     def run_on_file_or_subfiles_list(
-        self, file_or_subfiles_list: List[Union[str, List[str]]], *, clear: bool = True
+        self, file_or_subfiles_list: list[str | list[str]], *, clear: bool = True
     ) -> None:
         if clear:
             self.counters.clear()
@@ -146,7 +144,7 @@ def dump_values(self) -> None:
         if len(self.counters) == 0:
             raise ValueError("empty counter list")
 
-        value_tables: List[Dict[str, str]] = [
+        value_tables: list[dict[str, str]] = [
             counter.get_all_values(self.precision) for counter in self.counters
         ]
 

src/neosca/ns_lca/ns_lca_counter.py

@@ -6,9 +6,11 @@
 import random
 import shutil
 import sys
+from collections import OrderedDict
+from collections.abc import Sequence
 from math import log as _log
 from math import sqrt as _sqrt
-from typing import Dict, List, Literal, Optional, OrderedDict, Sequence, Tuple, Union
+from typing import Literal
 
 from neosca import DATA_DIR
 from neosca.ns_io import Ns_IO
@@ -59,7 +61,7 @@ class Ns_LCA_Counter:
         "AdvV": "adverb variation",
         "ModV": "modifier variation",
     }
-    DEFAULT_MEASURES: List[str] = [
+    DEFAULT_MEASURES: list[str] = [
         *(item + suffix for item in COUNT_ITEMS for suffix in ("types", "tokens")),
         *FREQ_ITEMS,
     ]
@@ -76,8 +78,8 @@ def __init__(
     ) -> None:
         self.file_path = file_path
 
-        self.count_table: Dict[str, List[str]] = {item: [] for item in self.COUNT_ITEMS}
-        self.freq_table: Dict[str, Optional[Union[int, float]]] = {item: None for item in self.FREQ_ITEMS}
+        self.count_table: dict[str, list[str]] = {item: [] for item in self.COUNT_ITEMS}
+        self.freq_table: dict[str, int | float | None] = {item: None for item in self.FREQ_ITEMS}
 
         word_data_path = DATA_DIR / self.WORDLIST_DATAFILE_MAP[wordlist]
         logging.debug(f"Loading {word_data_path}...")
@@ -130,18 +132,18 @@ def get_msttr(cls, lemma_sequence: Sequence[str], *, section_size: int):
         Mean Segmental TTR
         """
         sample_no: int = 0
-        msttr: Union[int, float] = 0
+        msttr: int | float = 0
         for chunk in chunks(lemma_sequence, section_size):
             if len(chunk) == section_size:
                 sample_no += 1
                 msttr += safe_div(len(set(chunk)), section_size)
         return safe_div(msttr, sample_no)
 
-    def determine_counts(self, lempos_tuples: Tuple[Tuple[str, str], ...]):
+    def determine_counts(self, lempos_tuples: tuple[tuple[str, str], ...]):
         filtered_lempos_tuples = tuple(
             filter(lambda lempos: not self.word_classifier.is_("misc", *lempos), lempos_tuples)
         )
-        self.count_table["word"] = list(next(zip(*filtered_lempos_tuples)))
+        self.count_table["word"] = list(next(zip(*filtered_lempos_tuples, strict=False)))
 
         for lemma, pos in filtered_lempos_tuples:
             is_sophisticated = False
@@ -198,7 +200,7 @@ def determine_counts(self, lempos_tuples: Tuple[Tuple[str, str], ...]):
                     self.count_table["sverb"].append(lemma)
                     logging.debug(f'Counted "{lemma}" as a sophisticated verb')
 
-    def determine_freqs(self, *, section_size: Optional[int] = None) -> None:
+    def determine_freqs(self, *, section_size: int | None = None) -> None:
         if section_size is None:
             section_size = self.section_size
 
@@ -269,11 +271,11 @@ def determine_freqs(self, *, section_size: Optional[int] = None) -> None:
         self.freq_table["AdvV"] = safe_div(adv_type_no, lex_token_no)
         self.freq_table["ModV"] = safe_div((adv_type_no + adj_type_no), lex_token_no)
 
-    def determine_all_values(self, lempos_tuples: Tuple[Tuple[str, str], ...]) -> None:
+    def determine_all_values(self, lempos_tuples: tuple[tuple[str, str], ...]) -> None:
         self.determine_counts(lempos_tuples)
         self.determine_freqs()
 
-    def get_value(self, key: str, /, precision: int = 4) -> Union[int, float]:
+    def get_value(self, key: str, /, precision: int = 4) -> int | float:
         if (trimmed_key := key.removesuffix("types").removesuffix("tokens")) in self.COUNT_ITEMS:
             if key.endswith("types"):
                 return len(set(self.count_table[trimmed_key]))
@@ -287,7 +289,7 @@ def get_value(self, key: str, /, precision: int = 4) -> Union[int, float]:
         else:
             raise ValueError(f"Unknown key: {key}")
 
-    def get_matches(self, key: str, /) -> List[str]:
+    def get_matches(self, key: str, /) -> list[str]:
         if (trimmed_key := key.removesuffix("types").removesuffix("tokens")) in self.COUNT_ITEMS:
             if key.endswith("types"):
                 return list(dict.fromkeys(self.count_table[trimmed_key]))