Skip to content

Commit 0ae1a23

Browse files
d-v-bd-v-b
committed
Merge branch 'feat/write-empty-chunks' of github.com:d-v-b/zarr-python into feat/write-empty-chunks
2 parents b7d117a + d21d3ce commit 0ae1a23

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

97 files changed

+5340
-3038
lines changed

‎.github/workflows/gpu_test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
strategy:
2626
matrix:
2727
python-version: ['3.11']
28-
numpy-version: ['2.0']
28+
numpy-version: ['2.1']
2929
dependency-set: ["minimal"]
3030

3131
steps:
@@ -55,7 +55,7 @@ jobs:
5555
cache: 'pip'
5656
- name: Install Hatch and CuPy
5757
run: |
58-
python -m pip install --upgrade pip
58+
python -m pip install --upgrade pip
5959
pip install hatch
6060
- name: Set Up Hatch Env
6161
run: |

‎.github/workflows/hypothesis.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
strategy:
2727
matrix:
2828
python-version: ['3.11']
29-
numpy-version: ['1.26']
29+
numpy-version: ['2.1']
3030
dependency-set: ["optional"]
3131

3232
steps:

‎.github/workflows/releases.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ jobs:
2323

2424
- name: Install PyBuild
2525
run: |
26-
python -m pip install --upgrade pip
26+
python -m pip install --upgrade pip
2727
pip install hatch
2828
- name: Build wheel and sdist
2929
run: hatch build
@@ -55,7 +55,7 @@ jobs:
5555
with:
5656
name: releases
5757
path: dist
58-
- uses: pypa/gh-action-pypi-publish@v1.10.3
58+
- uses: pypa/gh-action-pypi-publish@v1.12.3
5959
with:
6060
user: __token__
6161
password: ${{ secrets.pypi_password }}

‎.github/workflows/test.yml‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,32 @@ concurrency:
1616

1717
jobs:
1818
test:
19-
name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
19+
name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
2020

21-
runs-on: ubuntu-latest
2221
strategy:
2322
matrix:
2423
python-version: ['3.11', '3.12', '3.13']
25-
numpy-version: ['1.25', '1.26', '2.0']
24+
numpy-version: ['1.25', '2.1']
2625
dependency-set: ["minimal", "optional"]
26+
os: ["ubuntu-latest"]
27+
include:
28+
- python-version: '3.11'
29+
numpy-version: '1.25'
30+
dependency-set: 'optional'
31+
os: 'macos-latest'
32+
- python-version: '3.13'
33+
numpy-version: '2.1'
34+
dependency-set: 'optional'
35+
os: 'macos-latest'
36+
- python-version: '3.11'
37+
numpy-version: '1.25'
38+
dependency-set: 'optional'
39+
os: 'windows-latest'
40+
- python-version: '3.13'
41+
numpy-version: '2.1'
42+
dependency-set: 'optional'
43+
os: 'windows-latest'
44+
runs-on: ${{ matrix.os }}
2745

2846
steps:
2947
- uses: actions/checkout@v4
@@ -34,7 +52,7 @@ jobs:
3452
cache: 'pip'
3553
- name: Install Hatch
3654
run: |
37-
python -m pip install --upgrade pip
55+
python -m pip install --upgrade pip
3856
pip install hatch
3957
- name: Set Up Hatch Env
4058
run: |
@@ -66,7 +84,7 @@ jobs:
6684
cache: 'pip'
6785
- name: Install Hatch
6886
run: |
69-
python -m pip install --upgrade pip
87+
python -m pip install --upgrade pip
7088
pip install hatch
7189
- name: Set Up Hatch Env
7290
run: |
@@ -75,3 +93,22 @@ jobs:
7593
- name: Run Tests
7694
run: |
7795
hatch env run --env ${{ matrix.dependency-set }} run
96+
97+
test-complete:
98+
name: Test complete
99+
100+
needs:
101+
[
102+
test,
103+
test-upstream-and-min-deps,
104+
]
105+
if: always()
106+
runs-on: ubuntu-latest
107+
steps:
108+
- name: Check failure
109+
if: |
110+
contains(needs.*.result, 'failure') ||
111+
contains(needs.*.result, 'cancelled')
112+
run: exit 1
113+
- name: Success
114+
run: echo Success!

‎.pre-commit-config.yaml‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
ci:
22
autoupdate_commit_msg: "chore: update pre-commit hooks"
3+
autoupdate_schedule: "monthly"
34
autofix_commit_msg: "style: pre-commit fixes"
45
autofix_prs: false
56
default_stages: [pre-commit, pre-push]
6-
default_language_version:
7-
python: python3
87
repos:
98
- repo: https://github.com/astral-sh/ruff-pre-commit
10-
rev: v0.6.9
9+
rev: v0.8.2
1110
hooks:
1211
- id: ruff
1312
args: ["--fix", "--show-fixes"]
@@ -16,29 +15,27 @@ repos:
1615
rev: v2.3.0
1716
hooks:
1817
- id: codespell
19-
args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
18+
args: ["-L", "fo,ihs,kake,te", "-S", "fixture"]
2019
- repo: https://github.com/pre-commit/pre-commit-hooks
2120
rev: v5.0.0
2221
hooks:
2322
- id: check-yaml
23+
- id: trailing-whitespace
2424
- repo: https://github.com/pre-commit/mirrors-mypy
25-
rev: v1.11.2
25+
rev: v1.13.0
2626
hooks:
2727
- id: mypy
2828
files: src|tests
2929
additional_dependencies:
3030
# Package dependencies
31-
- asciitree
32-
- crc32c
31+
- packaging
3332
- donfig
34-
- numcodecs
35-
- numpy
33+
- numcodecs[crc32c]
34+
- numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved
3635
- typing_extensions
3736
- universal-pathlib
3837
# Tests
3938
- pytest
40-
# Zarr v2
41-
- types-redis
4239
- repo: https://github.com/scientific-python/cookie
4340
rev: 2024.08.19
4441
hooks:

‎README-v3.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ hatch env create test
3838
## Run the Tests
3939

4040
```
41-
hatch run test:run
41+
hatch run test:run
4242
```
4343

4444
or

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,13 @@
7878
</td>
7979
</tr>
8080
<tr>
81+
<td>Funding</td>
82+
<td>
83+
<a href="https://chanzuckerberg.com/eoss/">
84+
<img src="https://img.shields.io/badge/funded%20by-EOSS-FF414B.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0NSIgaGVpZ2h0PSI0NSI+PHBhdGggZD0iTTIyLjA5OCAyMS4wNzRjLTEuMjM1Ljg2LTIuNjggMS4zOTktNC42ODggMS40MzgtMi40MjYuMDUtNC4yMzgtMS41NTEtNC40MDYtMy45MThhNC40NjUgNC40NjUgMCAwIDEgMS4xODctMy4zOCA0LjQ4IDQuNDggMCAwIDEgMy4yODYtMS40NDQgNC42NzQgNC42NzQgMCAwIDEgMS44OTQuMzc1cy0uMTU2IDEuMzItLjIxIDEuOTE0bDEuOTg3LjAxNS4zNTYtMi45NzYtLjU2My0uMzU2YTYuNTQxIDYuNTQxIDAgMCAwLTMuNDg0LS45NjkgNi41OTMgNi41OTMgMCAwIDAtNC43NSAyLjA4NiA2LjQxNSA2LjQxNSAwIDAgMC0xLjcxNSA0Ljg3NWMuMTEzIDEuNjEuODA5IDMuMDc4IDEuOTUgNC4xNDEgMS4xNTYgMS4wNyAyLjcxNCAxLjY0NSA0LjM5OCAxLjYzMy4wMzkgMCAuMDc4IDAgLjEyNS0uMDA0YTkuOTE4IDkuOTE4IDAgMCAwIDMuNDMzLS43MjNsMS40MzQtMi43ODlzLS4wMjctLjA2Mi0uMjM0LjA3OCIgc3R5bGU9InN0cm9rZTpub25lO2ZpbGwtcnVsZTpub256ZXJvO2ZpbGw6I2ZmZjtmaWxsLW9wYWNpdHk6MSIvPjxwYXRoIGQ9Im0yOC44NjMgMjguMjE1LS4yNDIgMi43NDJ2LjAzMWMtLjEwMSAxLjAzNS0uNjUyIDEuOTE0LTEuNDg0IDIuMzUyLS42Ni4zNDctMS4zOC4zNDctMi4wMjgtLjAwOC0uNjc1LS4zNjMtLjk5Mi0uOTE4LTEuMTE3LTEuNjEzLS4xODctMS4xMDIuNDM4LTIuMjU0IDEuMjkzLTIuODMybDEzLjM1Mi04LjY4OGMuMDk3LjczOC4xNTIgMS40ODUuMTUyIDIuMjUgMCA4Ljk5Mi03LjMwOSAxNi4zMDUtMTYuMjkzIDE2LjMwNS04Ljk4OCAwLTE2LjI4OS03LjMxMy0xNi4yODktMTYuMzA1IDAtOC45ODggNy4zMDUtMTYuMyAxNi4yOTMtMTYuM3MxMi40OTYgNC4wOSAxNC45ODQgOS45MUwzOS4xMTQgMTVDMzYuMjYxIDguNjY0IDI5Ljg5MyA0LjIzNCAyMi41IDQuMjM0Yy03LjM5NSAwLTE4LjIxNSA4LjE3Mi0xOC4yMTUgMTguMjE1IDAgMTAuMDQ3IDguMTcyIDE4LjIxMSAxOC4yMTEgMTguMjExIDEwLjA0IDAgMTguMjE1LTguMTcyIDE4LjIxNS0xOC4yMSAwLTEwLjA0LS4xMS0yLjI5NC0uMzEzLTMuMzkxYTE5Ljk5NyAxOS45OTcgMCAwIDAtLjQ1My0xLjgzMmwtMy43OTMgMi4zNy01LjAyIDMuMThjLS4xNzUtLjY2OC0uNTgxLTEuMzQzLTEuNDQtMS43My0xLjAxMi0uNDY1LTIuNjYtLjExLTMuODY4LjU4MiAwIDAgMy4zMDUtNi40MDIgMy44Ni03LjQ1LjAzNS0uMDY2LS4wMTYtLjE0OC0uMDk4LS4xNDhoLTYuMThsLS4yNSAyLjA2N0gyNi41MDhsLTQuNzMgOS4wNjJjLS4wOTQuMTc2LjA5Ny4zNjMuMjc3LjI3NGwyLjUyMy0xLjM0YzEuMjE5LS42MjUgMy4yNS0xLjkxIDQuMjU0LTEuMTY4LjE0NS4xMS4zMzIuMzk4LjM0OC42OTVhLjM1Ny4zNTcgMCAwIDEtLjE0OS4yOTNsLTUuMDQzIDMuNDA2Yy0xLjYzNiAxLjE2OC0yLjI3NyAyLjkxLTIuMTIgNC41NTUuMTI0IDEuMzc1Ljk4NCAyLjU2NiAyLjI1NyAzLjI2MmE0LjE3IDQuMTcgMCAwIDAgMi4xMTMuNTE1IDQuMTY5IDQuMTY5IDAgMCAwIDEuODY0LS41YzEuNDM3LS43NTQgMi4zOTQtMi4yMzQgMi41NjItMy45NDVsLjQwMi00LjYxMy0yLjE5OSAxLjYzM1ptMCAwIiBzdHlsZT0ic3Ryb2tlOm5vbmU7ZmlsbC1ydWxlOm5vbnplcm87ZmlsbDojZmZmO2ZpbGwtb3BhY2l0eToxIi8+PC9zdmc+" alt="CZI's Essential Open Source Software for Science">
85+
</a>
86+
</td>
87+
</tr>
8188
<td>Citation</td>
8289
<td>
8390
<a href="https://doi.org/10.5281/zenodo.3773450">

‎bench/compress_normal.txt‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ Line # Hits Time Per Hit % Time Line Contents
1919
==============================================================
2020
137 def compress(source, char* cname, int clevel, int shuffle):
2121
138 """Compress data in a numpy array.
22-
139
22+
139
2323
140 Parameters
2424
141 ----------
2525
142 source : array-like
@@ -30,33 +30,33 @@ Line # Hits Time Per Hit % Time Line Contents
3030
147 Compression level.
3131
148 shuffle : int
3232
149 Shuffle filter.
33-
150
33+
150
3434
151 Returns
3535
152 -------
3636
153 dest : bytes-like
3737
154 Compressed data.
38-
155
38+
155
3939
156 """
40-
157
40+
157
4141
158 cdef:
4242
159 char *source_ptr
4343
160 char *dest_ptr
4444
161 Py_buffer source_buffer
4545
162 size_t nbytes, cbytes, itemsize
4646
163 200 506 2.5 0.2 array.array char_array_template = array.array('b', [])
4747
164 array.array dest
48-
165
48+
165
4949
166 # setup source buffer
5050
167 200 458 2.3 0.2 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS)
5151
168 200 119 0.6 0.0 source_ptr = <char *> source_buffer.buf
52-
169
52+
169
5353
170 # setup destination
5454
171 200 239 1.2 0.1 nbytes = source_buffer.len
5555
172 200 103 0.5 0.0 itemsize = source_buffer.itemsize
5656
173 200 2286 11.4 0.8 dest = array.clone(char_array_template, nbytes + BLOSC_MAX_OVERHEAD,
5757
174 zero=False)
5858
175 200 129 0.6 0.0 dest_ptr = <char *> dest.data.as_voidptr
59-
176
59+
176
6060
177 # perform compression
6161
178 200 1734 8.7 0.6 if _get_use_threads():
6262
179 # allow blosc to use threads internally
@@ -67,24 +67,24 @@ Line # Hits Time Per Hit % Time Line Contents
6767
184 cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes,
6868
185 source_ptr, dest_ptr,
6969
186 nbytes + BLOSC_MAX_OVERHEAD)
70-
187
70+
187
7171
188 else:
7272
189 with nogil:
7373
190 cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes,
7474
191 source_ptr, dest_ptr,
7575
192 nbytes + BLOSC_MAX_OVERHEAD, cname,
7676
193 0, 1)
77-
194
77+
194
7878
195 # release source buffer
7979
196 200 616 3.1 0.2 PyBuffer_Release(&source_buffer)
80-
197
80+
197
8181
198 # check compression was successful
8282
199 200 120 0.6 0.0 if cbytes <= 0:
8383
200 raise RuntimeError('error during blosc compression: %d' % cbytes)
84-
201
84+
201
8585
202 # resize after compression
8686
203 200 1896 9.5 0.6 array.resize(dest, cbytes)
87-
204
87+
204
8888
205 200 186 0.9 0.1 return dest
8989

9090
*******************************************************************************
@@ -100,19 +100,19 @@ Line # Hits Time Per Hit % Time Line Contents
100100
==============================================================
101101
75 def decompress(source, dest):
102102
76 """Decompress data.
103-
77
103+
77
104104
78 Parameters
105105
79 ----------
106106
80 source : bytes-like
107107
81 Compressed data, including blosc header.
108108
82 dest : array-like
109109
83 Object to decompress into.
110-
84
110+
84
111111
85 Notes
112112
86 -----
113113
87 Assumes that the size of the destination buffer is correct for the size of
114114
88 the uncompressed data.
115-
89
115+
89
116116
90 """
117117
91 cdef:
118118
92 int ret
@@ -122,7 +122,7 @@ Line # Hits Time Per Hit % Time Line Contents
122122
96 array.array source_array
123123
97 Py_buffer dest_buffer
124124
98 size_t nbytes
125-
99
125+
99
126126
100 # setup source buffer
127127
101 200 573 2.9 0.2 if PY2 and isinstance(source, array.array):
128128
102 # workaround fact that array.array does not support new-style buffer
@@ -134,13 +134,13 @@ Line # Hits Time Per Hit % Time Line Contents
134134
108 200 112 0.6 0.0 release_source_buffer = True
135135
109 200 144 0.7 0.1 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS)
136136
110 200 98 0.5 0.0 source_ptr = <char *> source_buffer.buf
137-
111
137+
111
138138
112 # setup destination buffer
139139
113 200 552 2.8 0.2 PyObject_GetBuffer(dest, &dest_buffer,
140140
114 PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
141141
115 200 100 0.5 0.0 dest_ptr = <char *> dest_buffer.buf
142142
116 200 84 0.4 0.0 nbytes = dest_buffer.len
143-
117
143+
117
144144
118 # perform decompression
145145
119 200 1856 9.3 0.8 if _get_use_threads():
146146
120 # allow blosc to use threads internally
@@ -149,12 +149,12 @@ Line # Hits Time Per Hit % Time Line Contents
149149
123 else:
150150
124 with nogil:
151151
125 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1)
152-
126
152+
126
153153
127 # release buffers
154154
128 200 754 3.8 0.3 if release_source_buffer:
155155
129 200 326 1.6 0.1 PyBuffer_Release(&source_buffer)
156156
130 200 165 0.8 0.1 PyBuffer_Release(&dest_buffer)
157-
131
157+
131
158158
132 # handle errors
159159
133 200 128 0.6 0.1 if ret <= 0:
160160
134 raise RuntimeError('error during blosc decompression: %d' % ret)

‎docs/conf.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
"sphinx_issues",
4848
"sphinx_copybutton",
4949
"sphinx_design",
50+
'sphinx_reredirects',
5051
]
5152

5253
issues_github_path = "zarr-developers/zarr-python"
@@ -81,6 +82,14 @@
8182
version = get_version("zarr")
8283
release = get_version("zarr")
8384

85+
redirects = {
86+
"spec": "https://zarr-specs.readthedocs.io",
87+
"spec/v1": 'https://zarr-specs.readthedocs.io/en/latest/v1/v1.0.html',
88+
"spec/v2": "https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html",
89+
"spec/v3": "https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html",
90+
"license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt"
91+
}
92+
8493
# The language for content autogenerated by Sphinx. Refer to documentation
8594
# for a list of supported languages.
8695
#

0 commit comments

Comments
 (0)