diff --git a/Cargo.lock b/Cargo.lock index 3bcce95..bc661c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -759,6 +759,16 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "om-file-format-sys" +version = "0.0.0" +source = "git+https://github.com/open-meteo/om-file-format?rev=9e8b847e5cd29f23871f74f5797f39b9ab6afa2f#9e8b847e5cd29f23871f74f5797f39b9ab6afa2f" +dependencies = [ + "bindgen", + "cc", + "libc", +] + [[package]] name = "om-file-format-sys" version = "0.0.0" @@ -772,14 +782,14 @@ dependencies = [ [[package]] name = "omfiles" version = "0.1.0" -source = "git+https://github.com/open-meteo/rust-omfiles?rev=91c7a1bfc01c5394332eb1b9b9a035f61554bf07#91c7a1bfc01c5394332eb1b9b9a035f61554bf07" +source = "git+https://github.com/open-meteo/rust-omfiles?rev=c7e6d4daed18e05bc8bb5d4373185096a493a662#c7e6d4daed18e05bc8bb5d4373185096a493a662" dependencies = [ "async-executor", "async-lock", "memmap2", "ndarray", "num-traits", - "om-file-format-sys", + "om-file-format-sys 0.0.0 (git+https://github.com/open-meteo/om-file-format?rev=9e8b847e5cd29f23871f74f5797f39b9ab6afa2f)", "thiserror", ] @@ -982,7 +992,7 @@ dependencies = [ "delegate", "num-traits", "numpy", - "om-file-format-sys", + "om-file-format-sys 0.0.0 (git+https://github.com/open-meteo/om-file-format?rev=a484a5a90914ff934fc451dff30b671b9ba8e38e)", "omfiles", "pyo3", "pyo3-async-runtimes", diff --git a/Cargo.toml b/Cargo.toml index 7e3b562..aaae57a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ async-lock = "3.4.0" numpy = "0.26" num-traits = "0.2" delegate = "0.13" -omfiles-rs = { git = "https://github.com/open-meteo/rust-omfiles", rev = "91c7a1bfc01c5394332eb1b9b9a035f61554bf07", package = "omfiles", features = ["metadata-tree"] } +omfiles-rs = { git = "https://github.com/open-meteo/rust-omfiles", rev = "c7e6d4daed18e05bc8bb5d4373185096a493a662", package = "omfiles", features = ["metadata-tree"] } om-file-format-sys = { git = "https://github.com/open-meteo/om-file-format", rev = "a484a5a90914ff934fc451dff30b671b9ba8e38e" } thiserror = "2.0.12" diff --git a/pyproject.toml b/pyproject.toml index 789e1df..1eb6a27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,9 @@ ignore = [ [project.entry-points."xarray.backends"] om = "omfiles.xarray:OmXarrayEntrypoint" +[project.entry-points."numcodecs.codecs"] +turbo_pfor = "omfiles._numcodecs:TurboPfor" + [project.entry-points."zarr.codecs"] "omfiles.pfor" = "omfiles._zarr3:PforCodec" "omfiles.pfor_serializer" = "omfiles._zarr3:PforSerializer" diff --git a/python/omfiles/_numcodecs.py b/python/omfiles/_numcodecs.py index 676e55d..53d4795 100644 --- a/python/omfiles/_numcodecs.py +++ b/python/omfiles/_numcodecs.py @@ -49,6 +49,3 @@ def decode(self, buf, out=None): # type: ignore buf = buf return self._impl.decode_array(buf, np.dtype(self.dtype), self.chunk_elements) - - -numcodecs.register_codec(TurboPfor) diff --git a/python/omfiles/omfiles.pyi b/python/omfiles/omfiles.pyi index 3f4d64e..6b2d676 100644 --- a/python/omfiles/omfiles.pyi +++ b/python/omfiles/omfiles.pyi @@ -184,6 +184,16 @@ class OmFileReader: It is safe to call this method multiple times. """ + def get_complete_lut(self) -> builtins.list[builtins.int]: + r""" + Retrieve the complete lookup table for the variable. + + The lookup table is a monotonically increasing array of u64 values containing + n+1 elements, where n is the number of chunks. Each value represents the absolute + offset in the file of the end of the chunk, and the first value is the start offset + of the first chunk. The size of chunk j can be calculated as lut[j+1] - lut[j] + using zero-based indexing. + """ def get_child_by_index(self, index: builtins.int) -> OmFileReader: r""" Get a child reader at the specified index. diff --git a/src/codecs.rs b/src/codecs.rs index f160437..7da54ae 100644 --- a/src/codecs.rs +++ b/src/codecs.rs @@ -2,6 +2,7 @@ use numpy::{ ndarray, IntoPyArray, PyArray1, PyArrayDescr, PyArrayDescrMethods, PyArrayDyn, PyArrayMethods, PyUntypedArray, PyUntypedArrayMethods, }; +use omfiles_rs::_om_file_format_sys as om_file_format_sys; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; diff --git a/src/reader.rs b/src/reader.rs index f1c540b..f0bdd22 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -356,6 +356,24 @@ impl OmFileReader { self.with_reader(|reader| Ok(reader.data_type().is_scalar())) } + /// Retrieve the complete lookup table for the variable. + /// + /// The lookup table is a monotonically increasing array of u64 values containing + /// n+1 elements, where n is the number of chunks. Each value represents the absolute + /// offset in the file of the end of the chunk, and the first value is the start offset + /// of the first chunk. The size of chunk j can be calculated as lut[j+1] - lut[j] + /// using zero-based indexing. + fn get_complete_lut(&self) -> PyResult> { + let lut = self.with_reader(|reader| { + reader + .expect_array() + .map_err(convert_omfilesrs_error)? + .get_complete_lut() + .map_err(convert_omfilesrs_error) + })?; + Ok(lut) + } + /// Check if the variable is a group (a variable with data type None). /// /// Returns: