WIP Basic labeled tensor functionality

TODO: Split Stack from commit

Author: ricardoV94

pytensor/xtensor/__init__.py

@@ -0,0 +1,12 @@
+import warnings
+
+import pytensor.xtensor.rewriting
+from pytensor.xtensor.type import (
+    XTensorType,
+    as_xtensor,
+    xtensor,
+    xtensor_constant,
+)
+
+
+warnings.warn("xtensor module is experimental and full of bugs")

pytensor/xtensor/basic.py

@@ -0,0 +1,86 @@
+from collections.abc import Sequence
+
+from pytensor.graph import Apply, Op
+from pytensor.tensor.type import TensorType
+from pytensor.xtensor.type import XTensorType, as_xtensor, xtensor
+
+
+class XOp(Op):
+    """A base class for XOps that shouldn't be materialized"""
+
+    def perform(self, node, inputs, outputs):
+        raise NotImplementedError(
+            f"xtensor operation {self} must be lowered to equivalent tensor operations"
+        )
+
+
+class XViewOp(Op):
+    # Make this a View Op with C-implementation
+    view_map = {0: [0]}
+
+    def perform(self, node, inputs, output_storage):
+        output_storage[0][0] = inputs[0]
+
+
+class TensorFromXTensor(XViewOp):
+    __props__ = ()
+
+    def make_node(self, x) -> Apply:
+        if not isinstance(x.type, XTensorType):
+            raise TypeError(f"x must be have an XTensorType, got {type(x.type)}")
+        output = TensorType(x.type.dtype, shape=x.type.shape)()
+        return Apply(self, [x], [output])
+
+
+tensor_from_xtensor = TensorFromXTensor()
+
+
+class XTensorFromTensor(XViewOp):
+    __props__ = ("dims",)
+
+    def __init__(self, dims: Sequence[str]):
+        super().__init__()
+        self.dims = tuple(dims)
+
+    def make_node(self, x) -> Apply:
+        if not isinstance(x.type, TensorType):
+            raise TypeError(f"x must be an TensorType type, got {type(x.type)}")
+        output = xtensor(dtype=x.type.dtype, dims=self.dims, shape=x.type.shape)
+        return Apply(self, [x], [output])
+
+
+def xtensor_from_tensor(x, dims):
+    return XTensorFromTensor(dims=dims)(x)
+
+
+class Rename(XViewOp):
+    __props__ = ("new_dims",)
+
+    def __init__(self, new_dims: tuple[str, ...]):
+        super().__init__()
+        self.new_dims = new_dims
+
+    def make_node(self, x):
+        x = as_xtensor(x)
+        output = x.type.clone(dims=self.new_dims)()
+        return Apply(self, [x], [output])
+
+
+def rename(x, name_dict: dict[str, str] | None = None, **names: str):
+    if name_dict is not None:
+        if names:
+            raise ValueError("Cannot use both positional and keyword names in rename")
+        names = name_dict
+
+    x = as_xtensor(x)
+    old_names = x.type.dims
+    new_names = list(old_names)
+    for old_name, new_name in names.items():
+        try:
+            new_names[old_names.index(old_name)] = new_name
+        except IndexError:
+            raise ValueError(
+                f"Cannot rename {old_name} to {new_name}: {old_name} not in {old_names}"
+            )
+
+    return Rename(tuple(new_names))(x)

pytensor/xtensor/readme.md

@@ -0,0 +1,69 @@
+# XTensor Module
+
+This module implements as abstraction layer on regular tensor operations, that behaves like Xarray.
+
+A new type `XTensorType`, generalizes the `TensorType` with the addition of a `dims` attribute, 
+that labels the dimensions of the tensor. 
+
+Variables of `XTensorType` (i.e.,  `XTensorVariable`s) are the symbolic counterpart to xarray DataArray objects.
+
+The module implements several PyTensor operations `XOp`s, whose signature mimics that of xarray (and xarray_einstants) DataArray operations.
+These operations, unlike most regular PyTensor operations, cannot be directly evaluated, but require a rewrite (lowering) into
+a regular tensor graph that can itself be evaluated as usual.
+
+Like regular PyTensor, we don't need an Op for every possible method or function in the public API of xarray.
+If the existing XOps can be composed to produce the desired result, then we can use them directly.
+
+## Coordinates
+For now, there's no analogous of xarray coordinates, so you won't be able to do coordinate operations like `.sel`.
+The graphs produced by an xarray program without coords are much more amenable to the numpy-like backend of PyTensor.
+Coords involve aspects of Pandas/database query and joining that are not trivially expressible in PyTensor.
+
+## Example
+
+```python
+import pytensor.tensor as pt
+import pytensor.xtensor as px
+
+a = pt.tensor("a", shape=(3,))
+b = pt.tensor("b", shape=(4,))
+
+ax = px.as_xtensor(a, dims=["x"])
+bx = px.as_xtensor(b, dims=["y"])
+
+zx = ax + bx
+assert zx.type == px.type.XTensorType("float64", dims=["x", "y"], shape=(3, 4))
+
+z = zx.values
+z.dprint()
+# TensorFromXTensor [id A]
+#  └─ XElemwise{scalar_op=Add()} [id B]
+#     ├─ XTensorFromTensor{dims=('x',)} [id C]
+#     │  └─ a [id D]
+#     └─ XTensorFromTensor{dims=('y',)} [id E]
+#        └─ b [id F]
+```
+
+Once we compile the graph, no `XOp`s are left.
+
+```python
+import pytensor
+
+with pytensor.config.change_flags(optimizer_verbose=True):
+    fn = pytensor.function([a, b], z)
+
+# rewriting: rewrite lower_elemwise replaces XElemwise{scalar_op=Add()}.0 of XElemwise{scalar_op=Add()}(XTensorFromTensor{dims=('x',)}.0, XTensorFromTensor{dims=('y',)}.0) with XTensorFromTensor{dims=('x', 'y')}.0 of XTensorFromTensor{dims=('x', 'y')}(Add.0)
+# rewriting: rewrite useless_tensor_from_xtensor replaces TensorFromXTensor.0 of TensorFromXTensor(XTensorFromTensor{dims=('x',)}.0) with a of None
+# rewriting: rewrite useless_tensor_from_xtensor replaces TensorFromXTensor.0 of TensorFromXTensor(XTensorFromTensor{dims=('y',)}.0) with b of None
+# rewriting: rewrite useless_tensor_from_xtensor replaces TensorFromXTensor.0 of TensorFromXTensor(XTensorFromTensor{dims=('x', 'y')}.0) with Add.0 of Add(ExpandDims{axis=1}.0, ExpandDims{axis=0}.0)
+
+fn.dprint()
+# Add [id A] 2
+#  ├─ ExpandDims{axis=1} [id B] 1
+#  │  └─ a [id C]
+#  └─ ExpandDims{axis=0} [id D] 0
+#     └─ b [id E]
+```
+
+
+

pytensor/xtensor/rewriting/__init__.py

@@ -0,0 +1,2 @@
+import pytensor.xtensor.rewriting.basic
+import pytensor.xtensor.rewriting.shape

pytensor/xtensor/rewriting/basic.py

@@ -0,0 +1,54 @@
+from pytensor.graph import node_rewriter
+from pytensor.xtensor.basic import (
+    Rename,
+    TensorFromXTensor,
+    XTensorFromTensor,
+    xtensor_from_tensor,
+)
+from pytensor.xtensor.rewriting.utils import register_xcanonicalize
+
+
+@register_xcanonicalize
+@node_rewriter(tracks=[TensorFromXTensor])
+def useless_tensor_from_xtensor(fgraph, node):
+    """TensorFromXTensor(XTensorFromTensor(x)) -> x"""
+    [x] = node.inputs
+    if x.owner and isinstance(x.owner.op, XTensorFromTensor):
+        return [x.owner.inputs[0]]
+
+
+@register_xcanonicalize
+@node_rewriter(tracks=[XTensorFromTensor])
+def useless_xtensor_from_tensor(fgraph, node):
+    """XTensorFromTensor(TensorFromXTensor(x)) -> x"""
+    [x] = node.inputs
+    if x.owner and isinstance(x.owner.op, TensorFromXTensor):
+        return [x.owner.inputs[0]]
+
+
+@register_xcanonicalize
+@node_rewriter(tracks=[TensorFromXTensor])
+def useless_tensor_from_xtensor_of_rename(fgraph, node):
+    """TensorFromXTensor(Rename(x)) -> TensorFromXTensor(x)"""
+    [renamed_x] = node.inputs
+    if renamed_x.owner and isinstance(renamed_x.owner.op, Rename):
+        [x] = renamed_x.owner.inputs
+        return node.op(x, return_list=True)
+
+
+@register_xcanonicalize
+@node_rewriter(tracks=[Rename])
+def useless_rename(fgraph, node):
+    """
+
+    Rename(Rename(x, inner_dims), outer_dims) -> Rename(x, outer_dims)
+    Rename(X, XTensorFromTensor(x, inner_dims), outer_dims) -> XTensorFrom_tensor(x, outer_dims)
+    """
+    [renamed_x] = node.inputs
+    if renamed_x.owner:
+        if isinstance(renamed_x.owner.op, Rename):
+            [x] = renamed_x.owner.inputs
+            return [node.op(x)]
+        elif isinstance(renamed_x.owner.op, TensorFromXTensor):
+            [x] = renamed_x.owner.inputs
+            return [xtensor_from_tensor(x, dims=node.op.new_dims)]

pytensor/xtensor/rewriting/shape.py

@@ -0,0 +1,29 @@
+from pytensor.graph import node_rewriter
+from pytensor.tensor import moveaxis
+from pytensor.xtensor.basic import tensor_from_xtensor, xtensor_from_tensor
+from pytensor.xtensor.rewriting.basic import register_xcanonicalize
+from pytensor.xtensor.shape import Stack
+
+
+@register_xcanonicalize
+@node_rewriter(tracks=[Stack])
+def lower_stack(fgraph, node):
+    [x] = node.inputs
+    batch_ndim = x.type.ndim - len(node.op.stacked_dims)
+    stacked_axes = [
+        i for i, dim in enumerate(x.type.dims) if dim in node.op.stacked_dims
+    ]
+    end = tuple(range(-len(stacked_axes), 0))
+
+    x_tensor = tensor_from_xtensor(x)
+    x_tensor_transposed = moveaxis(x_tensor, source=stacked_axes, destination=end)
+    if batch_ndim == (x.type.ndim - 1):
+        # This happens when we stack a "single" dimension, in this case all we need is the transpose
+        # Note: If we have meaningful rewrites before lowering, consider canonicalizing this as a Transpose + Rename
+        final_tensor = x_tensor_transposed
+    else:
+        final_shape = (*tuple(x_tensor_transposed.shape)[:batch_ndim], -1)
+        final_tensor = x_tensor_transposed.reshape(final_shape)
+
+    new_out = xtensor_from_tensor(final_tensor, dims=node.outputs[0].type.dims)
+    return [new_out]

pytensor/xtensor/rewriting/utils.py

@@ -0,0 +1,33 @@
+from pytensor.compile import optdb
+from pytensor.graph.rewriting.basic import NodeRewriter
+from pytensor.graph.rewriting.db import EquilibriumDB, RewriteDatabase
+
+
+optdb.register(
+    "xcanonicalize",
+    EquilibriumDB(ignore_newtrees=False),
+    "fast_run",
+    "fast_compile",
+    "xtensor",
+    position=0,
+)
+
+
+def register_xcanonicalize(
+    node_rewriter: RewriteDatabase | NodeRewriter | str, *tags: str, **kwargs
+):
+    if isinstance(node_rewriter, str):
+
+        def register(inner_rewriter: RewriteDatabase | NodeRewriter):
+            return register_xcanonicalize(
+                inner_rewriter, node_rewriter, *tags, **kwargs
+            )
+
+        return register
+
+    else:
+        name = kwargs.pop("name", None) or node_rewriter.__name__
+        optdb["xtensor"].register(
+            name, node_rewriter, "fast_run", "fast_compile", *tags, **kwargs
+        )
+        return node_rewriter

pytensor/xtensor/shape.py

@@ -0,0 +1,71 @@
+from collections.abc import Sequence
+
+from pytensor.graph import Apply
+from pytensor.xtensor.basic import XOp
+from pytensor.xtensor.type import as_xtensor, xtensor
+
+
+class Stack(XOp):
+    __props__ = ("new_dim_name", "stacked_dims")
+
+    def __init__(self, new_dim_name: str, stacked_dims: tuple[str, ...]):
+        super().__init__()
+        if new_dim_name in stacked_dims:
+            raise ValueError(
+                f"Stacking dim {new_dim_name} must not be in {stacked_dims}"
+            )
+        if not stacked_dims:
+            raise ValueError(f"Stacking dims must not be empty: got {stacked_dims}")
+        self.new_dim_name = new_dim_name
+        self.stacked_dims = stacked_dims
+
+    def make_node(self, x):
+        x = as_xtensor(x)
+        if not (set(self.stacked_dims) <= set(x.type.dims)):
+            raise ValueError(
+                f"Stacking dims {self.stacked_dims} must be a subset of {x.type.dims}"
+            )
+        if self.new_dim_name in x.type.dims:
+            raise ValueError(
+                f"Stacking dim {self.new_dim_name} must not be in {x.type.dims}"
+            )
+        if len(self.stacked_dims) == x.type.ndim:
+            batch_dims, batch_shape = (), ()
+        else:
+            batch_dims, batch_shape = zip(
+                *(
+                    (dim, shape)
+                    for dim, shape in zip(x.type.dims, x.type.shape)
+                    if dim not in self.stacked_dims
+                )
+            )
+        stack_shape = 1
+        for dim, shape in zip(x.type.dims, x.type.shape):
+            if dim in self.stacked_dims:
+                if shape is None:
+                    stack_shape = None
+                    break
+                else:
+                    stack_shape *= shape
+        output = xtensor(
+            dtype=x.type.dtype,
+            shape=(*batch_shape, stack_shape),
+            dims=(*batch_dims, self.new_dim_name),
+        )
+        return Apply(self, [x], [output])
+
+
+def stack(x, dim: dict[str, Sequence[str]] | None = None, **dims: Sequence[str]):
+    if dim is not None:
+        if dims:
+            raise ValueError("Cannot use both positional dim and keyword dims in stack")
+        dims = dim
+
+    y = x
+    for new_dim_name, stacked_dims in dims.items():
+        if isinstance(stacked_dims, str):
+            raise TypeError(
+                f"Stacking dims must be a sequence of strings, got a single string: {stacked_dims}"
+            )
+        y = Stack(new_dim_name, tuple(stacked_dims))(y)
+    return y