Implement basic labeled tensor functionality

Author: ricardoV94

.github/workflows/test.yml

@@ -82,11 +82,12 @@ jobs:
         install-numba: [0]
         install-jax: [0]
         install-torch: [0]
+        install-xarray: [0]
         part:
-          - "tests --ignore=tests/tensor --ignore=tests/scan --ignore=tests/sparse"
+          - "tests --ignore=tests/tensor --ignore=tests/scan --ignore=tests/sparse --ignore=tests/xtensor"
           - "tests/scan"
           - "tests/sparse"
-          - "tests/tensor --ignore=tests/tensor/conv --ignore=tests/tensor/rewriting --ignore=tests/tensor/test_math.py --ignore=tests/tensor/test_basic.py --ignore=tests/tensor/test_blas.py --ignore=tests/tensor/test_math_scipy.py --ignore=tests/tensor/test_inplace.py --ignore=tests/tensor/test_elemwise.py"
+          - "tests/tensor --ignore=tests/tensor/conv --ignore=tests/tensor/rewriting --ignore=tests/tensor/test_math.py --ignore=tests/tensor/test_basic.py --ignore=tests/tensor/test_inplace.py --ignore=tests/tensor/test_blas.py --ignore=tests/tensor/test_elemwise.py --ignore=tests/tensor/test_math_scipy.py"
           - "tests/tensor/conv"
           - "tests/tensor/rewriting"
           - "tests/tensor/test_math.py"
@@ -115,6 +116,7 @@ jobs:
             install-numba: 0
             install-jax: 0
             install-torch: 0
+            install-xarray: 0
           - install-numba: 1
             os: "ubuntu-latest"
             python-version: "3.10"
@@ -150,6 +152,13 @@ jobs:
             fast-compile: 0
             float32: 0
             part: "tests/link/pytorch"
+          - install-xarray: 1
+            os: "ubuntu-latest"
+            python-version: "3.13"
+            numpy-version: ">=2.0"
+            fast-compile: 0
+            float32: 0
+            part: "tests/xtensor"
           - os: macos-15
             python-version: "3.13"
             numpy-version: ">=2.0"
@@ -196,6 +205,7 @@ jobs:
           if [[ $INSTALL_NUMBA == "1" ]]; then micromamba install --yes -q -c conda-forge "python~=${PYTHON_VERSION}" "numba>=0.57"; fi
           if [[ $INSTALL_JAX == "1" ]]; then micromamba install --yes -q -c conda-forge "python~=${PYTHON_VERSION}" jax jaxlib numpyro && pip install tensorflow-probability; fi
           if [[ $INSTALL_TORCH == "1" ]]; then micromamba install --yes -q -c conda-forge "python~=${PYTHON_VERSION}" pytorch pytorch-cuda=12.1 "mkl<=2024.0" -c pytorch -c nvidia; fi
+          if [[ $INSTALL_XARRAY == "1" ]]; then micromamba install --yes -q -c conda-forge "python~=${PYTHON_VERSION}" xarray xarray-einstats; fi
           pip install pytest-sphinx
           
           pip install -e ./
@@ -212,6 +222,7 @@ jobs:
           INSTALL_NUMBA: ${{ matrix.install-numba }}
           INSTALL_JAX: ${{ matrix.install-jax }}
           INSTALL_TORCH: ${{ matrix.install-torch}}
+          INSTALL_XARRAY: ${{ matrix.install-xarray }}
           OS: ${{ matrix.os}}
 
       - name: Run tests

pytensor/compile/mode.py

@@ -67,6 +67,8 @@ def register_linker(name, linker):
 if not config.cxx:
     exclude = ["cxx_only"]
 OPT_NONE = RewriteDatabaseQuery(include=[], exclude=exclude)
+# Minimum set of rewrites needed to evaluate a function. This is needed for graphs with "dummy" Operations
+OPT_MINIMUM = RewriteDatabaseQuery(include=["minimum_compile"], exclude=exclude)
 # Even if multiple merge optimizer call will be there, this shouldn't
 # impact performance.
 OPT_MERGE = RewriteDatabaseQuery(include=["merge"], exclude=exclude)
@@ -77,6 +79,7 @@ def register_linker(name, linker):
 OPT_STABILIZE = RewriteDatabaseQuery(include=["fast_run"], exclude=exclude)
 OPT_STABILIZE.position_cutoff = 1.5000001
 OPT_NONE.name = "OPT_NONE"
+OPT_MINIMUM.name = "OPT_MINIMUM"
 OPT_MERGE.name = "OPT_MERGE"
 OPT_FAST_RUN.name = "OPT_FAST_RUN"
 OPT_FAST_RUN_STABLE.name = "OPT_FAST_RUN_STABLE"
@@ -95,6 +98,7 @@ def register_linker(name, linker):
     None: OPT_NONE,
     "None": OPT_NONE,
     "merge": OPT_MERGE,
+    "minimum_compile": OPT_MINIMUM,
     "o4": OPT_FAST_RUN,
     "o3": OPT_O3,
     "o2": OPT_O2,
@@ -191,6 +195,7 @@ def apply(self, fgraph):
     "merge1", MergeOptimizer(), "fast_run", "fast_compile", "merge", position=0
 )
 
+
 # After scan1 opt at 0.5 and before ShapeOpt at 1
 # This should only remove nodes.
 # The opt should not do anything that need shape inference.

pytensor/xtensor/__init__.py

@@ -0,0 +1,12 @@
+import warnings
+
+import pytensor.xtensor.rewriting
+from pytensor.xtensor.type import (
+    XTensorType,
+    as_xtensor,
+    xtensor,
+    xtensor_constant,
+)
+
+
+warnings.warn("xtensor module is experimental and full of bugs")

pytensor/xtensor/basic.py

@@ -0,0 +1,100 @@
+from collections.abc import Sequence
+
+from pytensor.compile.ops import TypeCastingOp
+from pytensor.graph import Apply, Op
+from pytensor.tensor.type import TensorType
+from pytensor.xtensor.type import XTensorType, as_xtensor, xtensor
+
+
+class XOp(Op):
+    """A base class for XOps that shouldn't be materialized"""
+
+    def perform(self, node, inputs, outputs):
+        raise NotImplementedError(
+            f"xtensor operation {self} must be lowered to equivalent tensor operations"
+        )
+
+
+class XTypeCastOp(TypeCastingOp):
+    """Base class for Ops that type cast between TensorType and XTensorType.
+
+    This is like a `ViewOp` but without the expectation the input and output have identical types.
+    """
+
+
+class TensorFromXTensor(XTypeCastOp):
+    __props__ = ()
+
+    def make_node(self, x):
+        if not isinstance(x.type, XTensorType):
+            raise TypeError(f"x must be have an XTensorType, got {type(x.type)}")
+        output = TensorType(x.type.dtype, shape=x.type.shape)()
+        return Apply(self, [x], [output])
+
+    def L_op(self, inputs, outs, g_outs):
+        [x] = inputs
+        [g_out] = g_outs
+        return [xtensor_from_tensor(g_out, dims=x.type.dims)]
+
+
+tensor_from_xtensor = TensorFromXTensor()
+
+
+class XTensorFromTensor(XTypeCastOp):
+    __props__ = ("dims",)
+
+    def __init__(self, dims: Sequence[str]):
+        super().__init__()
+        self.dims = tuple(dims)
+
+    def make_node(self, x):
+        if not isinstance(x.type, TensorType):
+            raise TypeError(f"x must be an TensorType type, got {type(x.type)}")
+        output = xtensor(dtype=x.type.dtype, dims=self.dims, shape=x.type.shape)
+        return Apply(self, [x], [output])
+
+    def L_op(self, inputs, outs, g_outs):
+        [g_out] = g_outs
+        return [tensor_from_xtensor(g_out)]
+
+
+def xtensor_from_tensor(x, dims, name=None):
+    return XTensorFromTensor(dims=dims)(x, name=name)
+
+
+class Rename(XTypeCastOp):
+    __props__ = ("new_dims",)
+
+    def __init__(self, new_dims: tuple[str, ...]):
+        super().__init__()
+        self.new_dims = new_dims
+
+    def make_node(self, x):
+        x = as_xtensor(x)
+        output = x.type.clone(dims=self.new_dims)()
+        return Apply(self, [x], [output])
+
+    def L_op(self, inputs, outs, g_outs):
+        [x] = inputs
+        [g_out] = g_outs
+        return [rename(g_out, dims=x.type.dims)]
+
+
+def rename(x, name_dict: dict[str, str] | None = None, **names: str):
+    if name_dict is not None:
+        if names:
+            raise ValueError("Cannot use both positional and keyword names in rename")
+        names = name_dict
+
+    x = as_xtensor(x)
+    old_names = x.type.dims
+    new_names = list(old_names)
+    for old_name, new_name in names.items():
+        try:
+            new_names[old_names.index(old_name)] = new_name
+        except ValueError:
+            raise ValueError(
+                f"Cannot rename {old_name} to {new_name}: {old_name} not in {old_names}"
+            )
+
+    return Rename(tuple(new_names))(x)

pytensor/xtensor/readme.md

@@ -0,0 +1,69 @@
+# XTensor Module
+
+This module implements as abstraction layer on regular tensor operations, that behaves like Xarray.
+
+A new type `XTensorType`, generalizes the `TensorType` with the addition of a `dims` attribute, 
+that labels the dimensions of the tensor. 
+
+Variables of `XTensorType` (i.e.,  `XTensorVariable`s) are the symbolic counterpart to xarray DataArray objects.
+
+The module implements several PyTensor operations `XOp`s, whose signature mimics that of xarray (and xarray_einstants) DataArray operations.
+These operations, unlike most regular PyTensor operations, cannot be directly evaluated, but require a rewrite (lowering) into
+a regular tensor graph that can itself be evaluated as usual.
+
+Like regular PyTensor, we don't need an Op for every possible method or function in the public API of xarray.
+If the existing XOps can be composed to produce the desired result, then we can use them directly.
+
+## Coordinates
+For now, there's no analogous of xarray coordinates, so you won't be able to do coordinate operations like `.sel`.
+The graphs produced by an xarray program without coords are much more amenable to the numpy-like backend of PyTensor.
+Coords involve aspects of Pandas/database query and joining that are not trivially expressible in PyTensor.
+
+## Example
+
+```python
+import pytensor.tensor as pt
+import pytensor.xtensor as px
+
+a = pt.tensor("a", shape=(3,))
+b = pt.tensor("b", shape=(4,))
+
+ax = px.as_xtensor(a, dims=["x"])
+bx = px.as_xtensor(b, dims=["y"])
+
+zx = ax + bx
+assert zx.type == px.type.XTensorType("float64", dims=["x", "y"], shape=(3, 4))
+
+z = zx.values
+z.dprint()
+# TensorFromXTensor [id A]
+#  └─ XElemwise{scalar_op=Add()} [id B]
+#     ├─ XTensorFromTensor{dims=('x',)} [id C]
+#     │  └─ a [id D]
+#     └─ XTensorFromTensor{dims=('y',)} [id E]
+#        └─ b [id F]
+```
+
+Once we compile the graph, no `XOp`s are left.
+
+```python
+import pytensor
+
+with pytensor.config.change_flags(optimizer_verbose=True):
+    fn = pytensor.function([a, b], z)
+
+# rewriting: rewrite lower_elemwise replaces XElemwise{scalar_op=Add()}.0 of XElemwise{scalar_op=Add()}(XTensorFromTensor{dims=('x',)}.0, XTensorFromTensor{dims=('y',)}.0) with XTensorFromTensor{dims=('x', 'y')}.0 of XTensorFromTensor{dims=('x', 'y')}(Add.0)
+# rewriting: rewrite useless_tensor_from_xtensor replaces TensorFromXTensor.0 of TensorFromXTensor(XTensorFromTensor{dims=('x',)}.0) with a of None
+# rewriting: rewrite useless_tensor_from_xtensor replaces TensorFromXTensor.0 of TensorFromXTensor(XTensorFromTensor{dims=('y',)}.0) with b of None
+# rewriting: rewrite useless_tensor_from_xtensor replaces TensorFromXTensor.0 of TensorFromXTensor(XTensorFromTensor{dims=('x', 'y')}.0) with Add.0 of Add(ExpandDims{axis=1}.0, ExpandDims{axis=0}.0)
+
+fn.dprint()
+# Add [id A] 2
+#  ├─ ExpandDims{axis=1} [id B] 1
+#  │  └─ a [id C]
+#  └─ ExpandDims{axis=0} [id D] 0
+#     └─ b [id E]
+```
+
+
+

pytensor/xtensor/rewriting/__init__.py

@@ -0,0 +1 @@
+import pytensor.xtensor.rewriting.basic

pytensor/xtensor/rewriting/basic.py

@@ -0,0 +1,62 @@
+from pytensor.graph import node_rewriter
+from pytensor.tensor.basic import register_infer_shape
+from pytensor.tensor.rewriting.basic import register_canonicalize, register_useless
+from pytensor.xtensor.basic import (
+    Rename,
+    TensorFromXTensor,
+    XTensorFromTensor,
+    xtensor_from_tensor,
+)
+from pytensor.xtensor.rewriting.utils import register_lower_xtensor
+
+
+@register_infer_shape
+@register_useless
+@register_canonicalize
+@register_lower_xtensor
+@node_rewriter(tracks=[TensorFromXTensor])
+def useless_tensor_from_xtensor(fgraph, node):
+    """TensorFromXTensor(XTensorFromTensor(x)) -> x"""
+    [x] = node.inputs
+    if x.owner and isinstance(x.owner.op, XTensorFromTensor):
+        return [x.owner.inputs[0]]
+
+
+@register_infer_shape
+@register_useless
+@register_canonicalize
+@register_lower_xtensor
+@node_rewriter(tracks=[XTensorFromTensor])
+def useless_xtensor_from_tensor(fgraph, node):
+    """XTensorFromTensor(TensorFromXTensor(x)) -> x"""
+    [x] = node.inputs
+    if x.owner and isinstance(x.owner.op, TensorFromXTensor):
+        return [x.owner.inputs[0]]
+
+
+@register_lower_xtensor
+@node_rewriter(tracks=[TensorFromXTensor])
+def useless_tensor_from_xtensor_of_rename(fgraph, node):
+    """TensorFromXTensor(Rename(x)) -> TensorFromXTensor(x)"""
+    [renamed_x] = node.inputs
+    if renamed_x.owner and isinstance(renamed_x.owner.op, Rename):
+        [x] = renamed_x.owner.inputs
+        return node.op(x, return_list=True)
+
+
+@register_lower_xtensor
+@node_rewriter(tracks=[Rename])
+def useless_rename(fgraph, node):
+    """
+
+    Rename(Rename(x, inner_dims), outer_dims) -> Rename(x, outer_dims)
+    Rename(X, XTensorFromTensor(x, inner_dims), outer_dims) -> XTensorFrom_tensor(x, outer_dims)
+    """
+    [renamed_x] = node.inputs
+    if renamed_x.owner:
+        if isinstance(renamed_x.owner.op, Rename):
+            [x] = renamed_x.owner.inputs
+            return [node.op(x)]
+        elif isinstance(renamed_x.owner.op, TensorFromXTensor):
+            [x] = renamed_x.owner.inputs
+            return [xtensor_from_tensor(x, dims=node.op.new_dims)]

pytensor/xtensor/rewriting/utils.py

@@ -0,0 +1,41 @@
+from pytensor.compile import optdb
+from pytensor.graph.rewriting.basic import NodeRewriter
+from pytensor.graph.rewriting.db import EquilibriumDB, RewriteDatabase
+
+
+lower_xtensor_db = EquilibriumDB(ignore_newtrees=False)
+
+optdb.register(
+    "lower_xtensor",
+    lower_xtensor_db,
+    "fast_run",
+    "fast_compile",
+    "minimum_compile",
+    position=0.1,
+)
+
+
+def register_lower_xtensor(
+    node_rewriter: RewriteDatabase | NodeRewriter | str, *tags: str, **kwargs
+):
+    if isinstance(node_rewriter, str):
+
+        def register(inner_rewriter: RewriteDatabase | NodeRewriter):
+            return register_lower_xtensor(
+                inner_rewriter, node_rewriter, *tags, **kwargs
+            )
+
+        return register
+
+    else:
+        name = kwargs.pop("name", None) or node_rewriter.__name__  # type: ignore
+        lower_xtensor_db.register(
+            name,
+            node_rewriter,
+            "fast_run",
+            "fast_compile",
+            "minimum_compile",
+            *tags,
+            **kwargs,
+        )
+        return node_rewriter