Cleanup, add documentation

Author: lshaw8317

ADD_LAZYFUNCS.md

@@ -0,0 +1,17 @@
+# Adding (lazy) functions
+
+Once you have written a (public API) function in Blosc2, it is important to:
+* Import it from the relevant module in the ``__init__.py`` file
+* Add it to the list of functions in ``__all__`` in the ``__init__.py`` file
+* If it is present in numpy, add it to the relevant dictionary (``local_ufunc_map``, ``ufunc_map`` ``ufunc_map_1param``) in ``ndarray.py``
+
+Finally, you also need to deal with it correctly within ``shape_utils.py``.
+
+If the function does not change the shape of the output, simply add it to ``elementwise_funcs`` and you're done.
+
+If the function _does_ change the shape of the output, it is likely either a reduction, a constructor, or a linear algebra function and so should be added to one of those lists (``reducers``, ``constructor`` or ``linalg_funcs``). If the function is a reduction, unless you need to handle an argument that is neither ``axis`` nor ``keepdims``, you don't need to do anything else.
+If your function is a constructor, you need to ensure it is handled within the ``visit_Call`` function appropriately (if it has a shape argument this is easy, just add it to the list of functions that has ``zeros, zeros_like`` etc.).
+
+For linear algebra functions it is likely you will have to write a bespoke shape handler within the ``linalg_shape`` function. There is also a list ``linalg_attrs`` for attributes which change the shape (currently only ``T`` and ``mT``) should you need to add one. You will probably need to edit the ``validation_patterns`` list at the top of the ``lazyexpr.py`` file to handle these attributes. Just extend the part that has the negative lookahead "(?!real|imag|T|mT|(".
+
+After this, the imports at the top of the ``lazyexpr.py`` should handle things, where an ``eager_funcs`` list is defined to handle eager execution of functions which change the output shape. Finally, in order to handle name changes between NumPy versions 1 and 2, it may be necessary to add aliases for functions within the blocks defined by ``if NUMPY_GE_2_0:`` in ``lazyexpr.py`` and ``ndarray.py``.

src/blosc2/lazyexpr.py

@@ -51,7 +51,7 @@
     ufunc_map_1param,
 )
 
-from .shape_utils import constructors, elementwise_funcs, infer_shape, lin_alg_attrs, lin_alg_funcs, reducers
+from .shape_utils import constructors, elementwise_funcs, infer_shape, linalg_attrs, linalg_funcs, reducers
 
 if not blosc2.IS_WASM:
     import numexpr
@@ -157,9 +157,9 @@ def ne_evaluate(expression, local_dict=None, **kwargs):
     "S": np.str_,
     "V": np.bytes_,
 }
-blosc2_funcs = constructors + lin_alg_funcs + elementwise_funcs + reducers
+blosc2_funcs = constructors + linalg_funcs + elementwise_funcs + reducers
 # functions that have to be evaluated before chunkwise lazyexpr machinery
-eager_funcs = lin_alg_funcs + reducers + ["slice"] + lin_alg_attrs
+eager_funcs = linalg_funcs + reducers + ["slice"] + ["." + attr for attr in linalg_attrs]
 # Gather all callable functions in numpy
 numpy_funcs = {
     name
@@ -569,7 +569,7 @@ def compute_smaller_slice(larger_shape, smaller_shape, larger_slice):
 validation_patterns = [
     r"[\;]",  # Flow control characters
     r"(^|[^\w])__[\w]+__($|[^\w])",  # Dunder methods
-    r"\.\b(?!real|imag|(\d*[eE]?[+-]?\d+)|(\d*[eE]?[+-]?\d+j)|\d*j\b|(sum|prod|min|max|std|mean|var|any|all|where)"
+    r"\.\b(?!real|imag|T|mT|(\d*[eE]?[+-]?\d+)|(\d*[eE]?[+-]?\d+j)|\d*j\b|(sum|prod|min|max|std|mean|var|any|all|where)"
     r"\s*\([^)]*\)|[a-zA-Z_]\w*\s*\([^)]*\))",  # Attribute patterns
 ]
 
@@ -595,10 +595,8 @@ def compute_smaller_slice(larger_shape, smaller_shape, larger_slice):
 valid_methods |= {"float32", "float64", "complex64", "complex128"}
 valid_methods |= {"bool", "str", "bytes"}
 valid_methods |= {
-    name
-    for name in dir(blosc2.NDArray)
-    if callable(getattr(blosc2.NDArray, name)) and not name.startswith("_")
-}
+    name for name in dir(blosc2.NDArray) if not name.startswith("_")
+}  # allow attributes and methods
 
 
 def validate_expr(expr: str) -> None:
@@ -621,9 +619,7 @@ def validate_expr(expr: str) -> None:
     # Check for forbidden patterns
     forbiddens = _blacklist_re.search(skip_quotes)
     if forbiddens is not None:
-        i = forbiddens.span()[0]
-        if expr[i : i + 2] != ".T" and expr[i : i + 3] != ".mT":  # allow tranpose methods
-            raise ValueError(f"'{expr}' is not a valid expression.")
+        raise ValueError(f"'{expr}' is not a valid expression.")
 
     # Check for invalid characters not covered by the tokenizer
     invalid_chars = re.compile(r"[^\w\s+\-*/%()[].,=<>!&|~^]")

src/blosc2/shape_utils.py

@@ -79,7 +79,7 @@
     "zeros_like",
 ]
 
-lin_alg_funcs = [
+linalg_funcs = [
     "concat",
     "diagonal",
     "expand_dims",
@@ -94,7 +94,7 @@
     "vecdot",
 ]
 
-lin_alg_attrs = ["T", "mT"]
+linalg_attrs = ["T", "mT"]
 reducers = ["sum", "prod", "min", "max", "std", "mean", "var", "any", "all", "count_nonzero"]
 
 # All the available constructors and reducers necessary for the (string) expression evaluator
@@ -317,7 +317,7 @@ def elementwise(*args):
 
 
 # --- Function registry ---
-FUNCTIONS = {  # ignore out arg
+REDUCTIONS = {  # ignore out arg
     func: lambda x, axis=None, keepdims=False, out=None: reduce_shape(x, axis, keepdims)
     for func in reducers
     # any unknown function will default to elementwise
@@ -391,7 +391,7 @@ def visit_Call(self, node):  # noqa : C901
                 kwargs[kw.arg] = self._lookup_value(kw.value)
 
         # ------- handle linear algebra ---------------
-        if base_name in lin_alg_funcs:
+        if base_name in linalg_funcs:
             return linalg_shape(base_name, args, kwargs)
 
         # ------- handle constructors ---------------
@@ -484,8 +484,8 @@ def visit_Call(self, node):  # noqa : C901
                 slices = [self._eval_slice(slice_arg)]
             return slice_shape(obj_shape, slices)
 
-        if base_name in FUNCTIONS:
-            return FUNCTIONS[base_name](*args, **kwargs)
+        if base_name in REDUCTIONS:
+            return REDUCTIONS[base_name](*args, **kwargs)
 
         shapes = [s for s in args if s is not None]
         if base_name not in elementwise_funcs: