Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions examples/adbc_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@

import adbc_driver_duckdb.dbapi
import pyarrow
from substrait.builders.plan import read_named_table, filter
from substrait.builders.extended_expression import scalar_function, column, literal
import pyarrow.substrait as pa_substrait

from substrait.builders.extended_expression import column, literal, scalar_function
from substrait.builders.plan import filter, read_named_table
from substrait.builders.type import i64
from substrait.extension_registry import ExtensionRegistry
import pyarrow.substrait as pa_substrait

registry = ExtensionRegistry()

Expand Down
20 changes: 10 additions & 10 deletions examples/builder_example.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import substrait.gen.proto.algebra_pb2 as stalg
from substrait.builders.extended_expression import (
column,
literal,
scalar_function,
)
from substrait.builders.plan import (
fetch,
filter,
read_named_table,
select,
filter,
sort,
fetch,
)
from substrait.builders.extended_expression import (
column,
scalar_function,
literal,
)
from substrait.builders.type import i64, boolean, struct, named_struct, fp64, string
from substrait.builders.type import boolean, fp64, i64, named_struct, string, struct
from substrait.extension_registry import ExtensionRegistry
from substrait.utils.display import pretty_print_plan, pretty_print_expression
import substrait.gen.proto.algebra_pb2 as stalg
from substrait.utils.display import pretty_print_expression, pretty_print_plan

registry = ExtensionRegistry(load_default_extensions=True)

Expand Down
4 changes: 2 additions & 2 deletions examples/dataframe_example.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import substrait.dataframe as sdf
from substrait.builders.plan import read_named_table
from substrait.builders.type import i64, boolean, struct, named_struct
from substrait.builders.type import boolean, i64, named_struct, struct
from substrait.extension_registry import ExtensionRegistry
import substrait.dataframe as sdf

registry = ExtensionRegistry(load_default_extensions=True)

Expand Down
7 changes: 4 additions & 3 deletions examples/duckdb_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@


import duckdb
from substrait.builders.plan import read_named_table, select, filter
from substrait.builders.extended_expression import column, scalar_function, literal
import pyarrow.substrait as pa_substrait

from substrait.builders.extended_expression import column, literal, scalar_function
from substrait.builders.plan import filter, read_named_table, select
from substrait.builders.type import i32
from substrait.extension_registry import ExtensionRegistry
import pyarrow.substrait as pa_substrait

try:
duckdb.install_extension("substrait")
Expand Down
11 changes: 5 additions & 6 deletions examples/narwhals_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,13 @@
# ]
# ///

from substrait.builders.plan import read_named_table
from substrait.builders.type import i64, boolean, struct, named_struct
from substrait.extension_registry import ExtensionRegistry

from narwhals.typing import FrameT
import narwhals as nw
import substrait.dataframe as sdf
from narwhals.typing import FrameT

import substrait.dataframe as sdf
from substrait.builders.plan import read_named_table
from substrait.builders.type import boolean, i64, named_struct, struct
from substrait.extension_registry import ExtensionRegistry

registry = ExtensionRegistry(load_default_extensions=True)

Expand Down
3 changes: 2 additions & 1 deletion examples/pyarrow_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.substrait as pa_substrait

import substrait
from substrait.builders.plan import select, read_named_table
from substrait.builders.plan import read_named_table, select

arrow_schema = pa.schema([pa.field("x", pa.int32()), pa.field("y", pa.int32())])

Expand Down
176 changes: 70 additions & 106 deletions pixi.lock

Large diffs are not rendered by default.

5 changes: 2 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,13 @@ respect-gitignore = true
target-version = "py310"
# never autoformat upstream or generated code
exclude = ["third_party/", "src/substrait/gen"]
lint.extend-select = ["I"]


[tool.pixi.workspace]
channels = ["conda-forge"]
platforms = ["win-64", "linux-64", "osx-64", "osx-arm64", "linux-aarch64"]

[tool.pixi.pypi-dependencies]
substrait = { path = ".", editable = true }

[tool.pixi.environments]
dev = { features = ["dev"], solve-group = "default" }
extensions = { features = ["extensions"], solve-group = "default" }
Expand Down
2 changes: 1 addition & 1 deletion src/substrait/builders/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
)
from substrait.extension_registry import ExtensionRegistry
from substrait.gen.proto.extensions.extensions_pb2 import AdvancedExtension
from substrait.gen.version import substrait_version
from substrait.type_inference import infer_plan_schema
from substrait.utils import (
merge_extension_declarations,
merge_extension_uris,
merge_extension_urns,
)
from substrait.gen.version import substrait_version

UnboundPlan = Callable[[ExtensionRegistry], stp.Plan]

Expand Down
1 change: 0 additions & 1 deletion src/substrait/dataframe/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import substrait.dataframe
from substrait.builders.extended_expression import column

from substrait.dataframe.dataframe import DataFrame
from substrait.dataframe.expression import Expression

Expand Down
3 changes: 2 additions & 1 deletion src/substrait/dataframe/dataframe.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from typing import Union, Iterable
from typing import Iterable, Union

import substrait.dataframe
from substrait.builders.plan import select
from substrait.dataframe.expression import Expression
Expand Down
6 changes: 3 additions & 3 deletions src/substrait/dataframe/expression.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import substrait.gen.proto.extended_expression_pb2 as stee
import substrait.gen.proto.type_pb2 as stp
from substrait.builders.extended_expression import (
UnboundExtendedExpression,
ExtendedExpressionOrUnbound,
UnboundExtendedExpression,
resolve_expression,
scalar_function,
)
import substrait.gen.proto.type_pb2 as stp
import substrait.gen.proto.extended_expression_pb2 as stee
from substrait.extension_registry import ExtensionRegistry


Expand Down
10 changes: 5 additions & 5 deletions src/substrait/extension_registry/__init__.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
"""Extension Registry module."""

from .registry import ExtensionRegistry
from .exceptions import UnhandledParameterizedTypeError, UnrecognizedSubstraitTypeError
from .function_entry import FunctionEntry, FunctionType
from .registry import ExtensionRegistry
from .signature_checker_helpers import (
normalize_substrait_type_names,
_check_integer_constraint,
types_equal,
_bind_type_parameter,
_check_integer_constraint,
covers,
normalize_substrait_type_names,
types_equal,
)
from .exceptions import UnrecognizedSubstraitTypeError, UnhandledParameterizedTypeError

__all__ = [
"ExtensionRegistry",
Expand Down
4 changes: 2 additions & 2 deletions src/substrait/extension_registry/function_entry.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
from enum import Enum
from typing import Optional, Union

from substrait.gen.json import simple_extensions as se
from substrait.derivation_expression import _parse, evaluate
from substrait.gen.json import simple_extensions as se
from substrait.gen.proto.type_pb2 import Type

from .signature_checker_helpers import normalize_substrait_type_names, covers
from .signature_checker_helpers import covers, normalize_substrait_type_names


class FunctionType(Enum):
Expand Down
3 changes: 1 addition & 2 deletions src/substrait/extension_registry/registry.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Extension Registry class."""

import re
import itertools
import re
from collections import defaultdict
from importlib.resources import files as importlib_files
from pathlib import Path
Expand All @@ -15,7 +15,6 @@

from .function_entry import FunctionEntry, FunctionType


# Constants
DEFAULT_URN_PREFIX = "https://github.com/substrait-io/substrait/blob/main/extensions"
# Format: extension:<organization>:<name>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from substrait.derivation_expression import _evaluate
from substrait.gen.antlr.SubstraitTypeParser import SubstraitTypeParser
from substrait.gen.proto.type_pb2 import Type
from .exceptions import UnrecognizedSubstraitTypeError, UnhandledParameterizedTypeError

from .exceptions import UnhandledParameterizedTypeError, UnrecognizedSubstraitTypeError

# Type aliases
TypeParameterMapping = Dict[str, object]
Expand Down
5 changes: 3 additions & 2 deletions src/substrait/proto.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ def _load():
substrait.gen.proto.extensions.extensions_pb2.SimpleExtensionDeclaration
becomes substrait.proto.SimpleExtensionDeclaration
"""
import sys
import importlib
import inspect
import pkgutil
import importlib
import sys

from substrait.gen import proto as _proto

selfmodule = sys.modules[__name__]
Expand Down
3 changes: 2 additions & 1 deletion src/substrait/simple_extension_utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from substrait.gen.json import simple_extensions as se
from typing import Union

from substrait.gen.json import simple_extensions as se


def build_arg(d: dict) -> Union[se.ValueArg, se.TypeArg, se.EnumerationArg]:
if "value" in d:
Expand Down
24 changes: 13 additions & 11 deletions src/substrait/sql/sql_to_substrait.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,31 @@
import random
import string
from typing import Callable

from deepdiff import DeepDiff
from sqloxide import parse_sql

from substrait.builders.extended_expression import (
UnboundExtendedExpression,
aggregate_function,
column,
scalar_function,
literal,
aggregate_function,
scalar_function,
window_function,
)
from substrait.builders.plan import (
aggregate,
fetch,
filter,
join,
read_named_table,
select,
filter,
sort,
fetch,
set,
join,
aggregate,
sort,
)
from substrait.gen.proto import type_pb2 as stt
from substrait.gen.proto import algebra_pb2 as stalg
from substrait.extension_registry import ExtensionRegistry
from typing import Callable
from deepdiff import DeepDiff
from substrait.gen.proto import algebra_pb2 as stalg
from substrait.gen.proto import type_pb2 as stt

SchemaResolver = Callable[[str], stt.NamedStruct]

Expand Down
5 changes: 3 additions & 2 deletions src/substrait/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@
Utility and debugging functions for Substrait.
"""

import substrait.gen.proto.type_pb2 as stp
import substrait.gen.proto.extensions.extensions_pb2 as ste
from typing import Iterable

import substrait.gen.proto.extensions.extensions_pb2 as ste
import substrait.gen.proto.type_pb2 as stp


def type_num_names(typ: stp.Type):
kind = typ.WhichOneof("kind")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import yaml

import substrait.gen.proto.algebra_pb2 as stalg
import substrait.gen.proto.type_pb2 as stt
import substrait.gen.proto.extended_expression_pb2 as stee
import substrait.gen.proto.extensions.extensions_pb2 as ste
import substrait.gen.proto.type_pb2 as stt
from substrait.builders.extended_expression import aggregate_function, literal
from substrait.extension_registry import ExtensionRegistry

Expand Down
2 changes: 1 addition & 1 deletion tests/builders/extended_expression/test_column.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import substrait.gen.proto.algebra_pb2 as stalg
import substrait.gen.proto.type_pb2 as stt
import substrait.gen.proto.extended_expression_pb2 as stee
import substrait.gen.proto.type_pb2 as stt
from substrait.builders.extended_expression import column

struct = stt.Type.Struct(
Expand Down
5 changes: 2 additions & 3 deletions tests/builders/extended_expression/test_if_then.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
import substrait.gen.proto.algebra_pb2 as stalg
import substrait.gen.proto.type_pb2 as stt
import substrait.gen.proto.extended_expression_pb2 as stee
import substrait.gen.proto.extensions.extensions_pb2 as ste
import substrait.gen.proto.type_pb2 as stt
from substrait.builders.extended_expression import (
column,
if_then,
literal,
scalar_function,
column,
)
from substrait.extension_registry import ExtensionRegistry


struct = stt.Type.Struct(
types=[
stt.Type(i64=stt.Type.I64(nullability=stt.Type.NULLABILITY_REQUIRED)),
Expand Down
3 changes: 2 additions & 1 deletion tests/builders/extended_expression/test_literal.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
from datetime import date

import substrait.gen.proto.algebra_pb2 as stalg
from substrait.builders.extended_expression import literal
from substrait.builders import type as sttb
from substrait.builders.extended_expression import literal


def extract_literal(builder):
Expand Down
5 changes: 3 additions & 2 deletions tests/builders/extended_expression/test_multi_or_list.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import substrait.gen.proto.algebra_pb2 as stalg
import substrait.gen.proto.type_pb2 as stt
import substrait.gen.proto.extended_expression_pb2 as stee
from substrait.builders.extended_expression import multi_or_list, literal
import substrait.gen.proto.type_pb2 as stt
from substrait.builders.extended_expression import literal, multi_or_list
from substrait.builders.type import i8
from substrait.extension_registry import ExtensionRegistry

Expand Down Expand Up @@ -85,6 +85,7 @@ def test_singular_or_list():

def test_multi_or_list_with_extension():
import yaml

import substrait.gen.proto.extensions.extensions_pb2 as ste
from substrait.builders.extended_expression import scalar_function

Expand Down
4 changes: 2 additions & 2 deletions tests/builders/extended_expression/test_scalar_function.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import yaml

import substrait.gen.proto.algebra_pb2 as stalg
import substrait.gen.proto.type_pb2 as stt
import substrait.gen.proto.extended_expression_pb2 as stee
import substrait.gen.proto.extensions.extensions_pb2 as ste
from substrait.builders.extended_expression import scalar_function, literal
import substrait.gen.proto.type_pb2 as stt
from substrait.builders.extended_expression import literal, scalar_function
from substrait.extension_registry import ExtensionRegistry

struct = stt.Type.Struct(
Expand Down
Loading
Loading