Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/mypy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,4 @@ jobs:

- name: Run mypy
run: |
python -m mypy cubed
python -m mypy src/cubed
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ MANIFEST
*.manifest
*.spec

# scikit-build-core dynamic version
src/cubed/version.py

# Installer logs
pip-log.txt
pip-delete-this-directory.txt
Expand Down
63 changes: 59 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
requires = ["hatchling>=1.17.1", "hatch-vcs"]
build-backend = "hatchling.build"

[project]
name = "cubed"
version = "0.21.0"
dynamic = ["version"]
authors = [
{name = "Tom White", email = "[email protected]"},
]
Expand Down Expand Up @@ -94,11 +94,15 @@ homepage = "https://github.com/cubed-dev/cubed"
documentation = "https://tomwhite.github.io/cubed"
repository = "https://github.com/cubed-dev/cubed"

[tool.hatch]
version.source = "vcs"
build.hooks.vcs.version-file = "src/cubed/version.py"

[tool.pytest.ini_options]
filterwarnings = [
"error::UserWarning",
]
testpaths = "cubed/tests"
testpaths = "src/cubed/tests"

[tool.ruff]
target-version = "py311"
Expand Down Expand Up @@ -145,3 +149,54 @@ known-third-party = [
"toolz",
"zarr",
]

[tool.mypy]
exclude = "tests|doc"
files = "src/"
show_error_codes = true
allow_redefinition = true

[[tool.mypy.overrides]]
# Ignore vendored libraries
modules = "mypy-cubed.vendor.*"
ignore_errors = true

[[tool.mypy.overrides]]
# Most of the numerical computing stack doesn't have type annotations yet.
module = [
"mypy-apache_beam.*",
"mypy-aiostream.*",
"mypy-array_api_compat.*",
"mypy-cloudpickle.*",
"mypy-coiled.*",
"mypy-dask.*",
"mypy-donfig.*",
"mypy-distributed.*",
"mypy-fsspec.*",
"mypy-icechunk.*",
"mypy-lithops.*",
"mypy-IPython.*",
"mypy-memray.*",
"mypy-modal.*",
"mypy-matplotlib.*",
"mypy-ndindex.*",
"mypy-networkx.*",
"mypy-numcodecs.*",
"mypy-numpy.*",
"mypy-obstore.*",
"mypy-pandas.*",
"mypy-psutil.*",
"mypy-pytest.*",
"mypy-ray.*",
"mypy-rechunker.*",
"mypy-seaborn.*",
"mypy-six.*",
"mypy-tenacity.*",
"mypy-tensorstore.*",
"mypy-tlz.*",
"mypy-toolz.*",
"mypy-tqdm.*",
"mypy-zarr.*",
"mypy-pyspark.*",
]
ignore_missing_imports = true
79 changes: 0 additions & 79 deletions setup.cfg

This file was deleted.

10 changes: 0 additions & 10 deletions setup.py

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from numbers import Number

import numpy as np
from tlz import first, frequencies
from tlz import frequencies

from ..utils import is_integer, parse_bytes

Expand Down Expand Up @@ -312,7 +312,7 @@ def auto_chunks(chunks, shape, limit, dtype, previous_chunks=None):
return tuple(chunks)

if limit is None:
limit = "128MiB" # config.get("array.chunk-size")
limit = "128MiB" # config.get("array.chunk-size")
if isinstance(limit, str):
limit = parse_bytes(limit)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,12 @@ from typing import List, Optional, Tuple
from cubed.types import T_Chunks, T_DType, T_RectangularChunks, T_Shape

def _check_regular_chunks(chunkset: T_RectangularChunks) -> bool: ...

def broadcast_chunks(
*chunkss
) -> Tuple[Tuple[int, ...]]: ...

def broadcast_chunks(*chunkss) -> Tuple[Tuple[int, ...]]: ...
def common_blockdim(blockdims: List[Tuple[int, ...]]) -> Tuple[int, ...]: ...

def normalize_chunks(
chunks: T_Chunks,
shape: Optional[T_Shape] = None,
limit: Optional[int] = None,
dtype: Optional[T_DType] = None,
previous_chunks: Optional[T_RectangularChunks] = None,
) -> T_RectangularChunks:
...
) -> T_RectangularChunks: ...
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -358,11 +358,6 @@ def lol_product(head, values):
return lol_product(head + (values[0],), values[1:])







def broadcast_dimensions(argpairs, numblocks, sentinels=(1, (1,)), consolidate=None):
"""Find block dimensions from arguments

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Core rechunking algorithm stuff."""

import logging
import warnings
from math import ceil, floor, prod
Expand Down Expand Up @@ -270,7 +271,6 @@ def multistage_rechunking_plan(

# increase the number of stages until min_mem is exceeded
for stage_count in range(1, MAX_STAGES):

stage_chunks = calculate_stage_chunks(read_chunks, write_chunks, stage_count)
pre_chunks = [read_chunks] + stage_chunks
post_chunks = stage_chunks + [write_chunks]
Expand All @@ -297,6 +297,7 @@ def multistage_rechunking_plan(
f"Consider decreasing min_mem ({min_mem}) or increasing "
f"({max_mem}) to find a more efficient plan.",
category=ExcessiveIOWarning,
stacklevel=2,
)
assert prev_plan is not None
return prev_plan
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Types definitions used by executors."""

from dataclasses import dataclass
from typing import (
Any,
Expand Down Expand Up @@ -61,12 +62,8 @@ class CopySpec(NamedTuple):


Config = Any # TODO: better typing for config
SingleArgumentStageFunction = Callable[
[Any, NamedArg(type=Any, name="config")], None
] # noqa: F821
NoArgumentStageFunction = Callable[
[NamedArg(type=Any, name="config")], None
] # noqa: F821
SingleArgumentStageFunction = Callable[[Any, NamedArg(type=Any, name="config")], None] # noqa: F821
NoArgumentStageFunction = Callable[[NamedArg(type=Any, name="config")], None] # noqa: F821
StageFunction = Union[NoArgumentStageFunction, SingleArgumentStageFunction]


Expand Down
Loading